Adopt ConfigKit for pipeline loading

This commit is contained in:
2026-09-07 03:54:41 +00:00
parent 0e6ba414b3
commit 9e40c729e1
33 changed files with 683 additions and 3162 deletions

View File

@@ -253,8 +253,8 @@ func TestAssembledSplitBundleCommandsUseOneEffectiveFamilyConfiguration(t *testi
}
var validated bytes.Buffer
if err := ConfigValidate(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath}, &validated); err != nil {
t.Fatalf("validate production default: %v", err)
if err := ConfigValidate(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath, "--profile", "production"}, &validated); err != nil {
t.Fatalf("validate production profile: %v", err)
}
if !strings.Contains(validated.String(), "profile=production") {
t.Fatalf("validate output = %q, want production profile", validated.String())
@@ -330,7 +330,7 @@ func TestAssembledSplitBundleCommandsUseOneEffectiveFamilyConfiguration(t *testi
}
func TestAssembledSplitBundleProfileSwitchRecordsProvenanceAndLimitsReuse(t *testing.T) {
production := loadAssembledSplitBundleConfig(t, "")
production := loadAssembledSplitBundleConfig(t, "production")
testingCfg := loadAssembledSplitBundleConfig(t, "testing")
workspace := t.TempDir()
production.Pipeline.Workspace.Root = workspace

View File

@@ -304,17 +304,9 @@ func TestConfigCommandsInspectMaintainedSplitBundleWithoutRuntimeState(t *testin
func writeInspectionProfiles(t *testing.T, pipelinePath, production, testing string) {
t.Helper()
data, err := os.ReadFile(pipelinePath)
if err != nil {
t.Fatal(err)
}
composition := "composition:\n default_profile: production\n profiles:\n production:\n overlay: production.yml\n testing:\n overlay: testing.yml\n"
if err := os.WriteFile(pipelinePath, append([]byte(composition), data...), 0o644); err != nil {
t.Fatal(err)
}
dir := filepath.Dir(pipelinePath)
mustWriteTestFile(t, filepath.Join(dir, "production.yml"), production)
mustWriteTestFile(t, filepath.Join(dir, "testing.yml"), testing)
profilesDir := filepath.Join(filepath.Dir(pipelinePath), "profiles")
mustWriteTestFile(t, filepath.Join(profilesDir, "production.yml"), production)
mustWriteTestFile(t, filepath.Join(profilesDir, "testing.yml"), testing)
}
func TestConfigCommandsSelectProfilesAndCampaigns(t *testing.T) {
@@ -326,7 +318,7 @@ func TestConfigCommandsSelectProfilesAndCampaigns(t *testing.T) {
if err := ConfigValidate(context.Background(), []string{"--config", pipelinePath}, &defaultOut); err != nil {
t.Fatalf("default ConfigValidate() error = %v", err)
}
if !strings.Contains(defaultOut.String(), "profile=production") {
if !strings.Contains(defaultOut.String(), "profile=none") {
t.Fatalf("default output = %q", defaultOut.String())
}
@@ -398,8 +390,9 @@ func TestConfigValidateReportsCanonicalPartyAndImportErrors(t *testing.T) {
dir := t.TempDir()
rootPath := filepath.Join(dir, "pipeline.yml")
mustWriteTestFile(t, rootPath, `composition:
imports: [conf.yml]
mustWriteTestFile(t, rootPath, `configkit:
imports:
- path: conf.yml
workspace:
root: /one
whisperx:
@@ -407,7 +400,7 @@ whisperx:
`)
mustWriteTestFile(t, filepath.Join(dir, "conf.yml"), "workspace:\n root: /two\n")
err := ConfigValidate(context.Background(), []string{"--config", rootPath}, io.Discard)
if err == nil || !strings.Contains(err.Error(), "workspace.root") || !strings.Contains(err.Error(), "pipeline.yml") || !strings.Contains(err.Error(), "conf.yml") {
if err == nil || !strings.Contains(err.Error(), "/workspace/root") || !strings.Contains(err.Error(), "pipeline.yml") || !strings.Contains(err.Error(), "conf.yml") {
t.Fatalf("import diagnostic = %v", err)
}
}
@@ -432,7 +425,7 @@ func TestConfigSourcesReportsStableOwnership(t *testing.T) {
for _, want := range []string{
"root: ",
"imports: none",
"profile: name=testing selection=cli overlay=",
"profile: name=testing selection=explicit overlay=",
"campaign: id=sample-campaign source=",
"party: source=",
"digest: ",
@@ -475,15 +468,7 @@ func TestConfigSourcesReportsCanonicalFamilyAndPublishOrigins(t *testing.T) {
func TestConfigSourcesPreservesProfileOwnershipThroughFamilyExpansion(t *testing.T) {
pipelinePath, campaignPath := writeInspectionFamilyConfig(t)
data, err := os.ReadFile(pipelinePath)
if err != nil {
t.Fatal(err)
}
profilePath := filepath.Join(filepath.Dir(pipelinePath), "testing.yml")
composition := "composition:\n default_profile: testing\n profiles:\n testing:\n overlay: testing.yml\n"
if err := os.WriteFile(pipelinePath, append([]byte(composition), data...), 0o644); err != nil {
t.Fatal(err)
}
profilePath := filepath.Join(filepath.Dir(pipelinePath), "profiles", "testing.yml")
mustWriteTestFile(t, profilePath, `scriptorium:
artifact_families:
character_note:
@@ -492,7 +477,7 @@ func TestConfigSourcesPreservesProfileOwnershipThroughFamilyExpansion(t *testing
`)
var out bytes.Buffer
if err := ConfigSources(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath}, &out); err != nil {
if err := ConfigSources(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath, "--profile", "testing"}, &out); err != nil {
t.Fatalf("ConfigSources() error = %v", err)
}
want := "scriptorium.artifacts.character_note_arannis.enabled\tfamily\t" + profilePath
@@ -505,13 +490,10 @@ func TestConfigSourcesReportsRootImportProfileAndDefaultOwnership(t *testing.T)
dir := t.TempDir()
rootPath := filepath.Join(dir, "pipeline.yml")
importPath := filepath.Join(dir, "base.yml")
profilePath := filepath.Join(dir, "testing.yml")
mustWriteTestFile(t, rootPath, `composition:
imports: [base.yml]
default_profile: testing
profiles:
testing:
overlay: testing.yml
profilePath := filepath.Join(dir, "profiles", "testing.yml")
mustWriteTestFile(t, rootPath, `configkit:
imports:
- path: base.yml
workspace:
root: /srv/narratio
whisperx:
@@ -521,13 +503,13 @@ whisperx:
mustWriteTestFile(t, profilePath, "whisperx:\n language: fr\n")
var out bytes.Buffer
if err := ConfigSources(context.Background(), []string{"--config", rootPath}, &out); err != nil {
if err := ConfigSources(context.Background(), []string{"--config", rootPath, "--profile", "testing"}, &out); err != nil {
t.Fatalf("ConfigSources() error = %v", err)
}
output := out.String()
for _, want := range []string{
"import: " + importPath,
"profile: name=testing selection=default overlay=" + profilePath,
"profile: name=testing selection=explicit overlay=" + profilePath,
"workspace.root\troot\t" + rootPath,
"storage.backend\timport\t" + importPath,
"whisperx.language\tprofile\t" + profilePath,

View File

@@ -37,10 +37,10 @@ func TestRunProfileSelectionFlowsThroughSharedLoader(t *testing.T) {
t.Fatal(err)
}
if language != "en" {
t.Fatalf("default profile language = %q, want en", language)
t.Fatalf("base configuration language = %q, want en", language)
}
for _, profile := range []string{"unknown", ""} {
for _, profile := range []string{"unknown"} {
err := runWithPipelineExecutor(context.Background(), []string{
"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--profile", profile,
}, io.Discard, executor)
@@ -48,19 +48,20 @@ func TestRunProfileSelectionFlowsThroughSharedLoader(t *testing.T) {
t.Fatalf("profile %q error = %v, want selection rejection", profile, err)
}
}
if err := runWithPipelineExecutor(context.Background(), []string{
"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--profile", "",
}, io.Discard, executor); err != nil {
t.Fatalf("explicit empty profile should select base: %v", err)
}
}
func enableCommandTestProfiles(t *testing.T, pipelinePath string) {
t.Helper()
data, err := os.ReadFile(pipelinePath)
if err != nil {
dir := filepath.Join(filepath.Dir(pipelinePath), "profiles")
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
composition := "composition:\n default_profile: production\n profiles:\n production:\n overlay: production.yml\n testing:\n overlay: testing.yml\n"
if err := os.WriteFile(pipelinePath, append([]byte(composition), data...), 0o644); err != nil {
t.Fatal(err)
}
dir := filepath.Dir(pipelinePath)
if err := os.WriteFile(filepath.Join(dir, "production.yml"), []byte("whisperx:\n language: en\n"), 0o644); err != nil {
t.Fatal(err)
}

View File

@@ -8,6 +8,15 @@ import (
"testing"
)
func absolutePath(t *testing.T, path string) string {
t.Helper()
absolute, err := filepath.Abs(path)
if err != nil {
t.Fatal(err)
}
return absolute
}
func TestArtifactFamiliesExpandCanonicalCharactersDeterministically(t *testing.T) {
dir := t.TempDir()
writePartyResolutionFile(t, dir+"/party.yml", `schema_version: narratio.party.v1
@@ -148,7 +157,7 @@ func TestArtifactFamilyStrictDecodeRejectsUnknownFields(t *testing.T) {
dir := t.TempDir()
pipelinePath, _, _ := writeArtifactFamilyConfig(t, dir, artifactFamilyPipelineYAML(validFamilyYAML+" unsupported: true\n"))
_, err := LoadPipeline(pipelinePath)
if err == nil || !strings.Contains(err.Error(), "field unsupported not found") {
if err == nil || !strings.Contains(err.Error(), "decode.unknown_field") {
t.Fatalf("LoadPipeline() error = %v", err)
}
}

View File

@@ -1,783 +0,0 @@
package config
import (
"bytes"
"encoding/json"
"fmt"
"io"
"sort"
"strconv"
"strings"
"gopkg.in/yaml.v3"
)
// compositionDocument is the presence-aware representation used while
// assembling pipeline configuration sources. It deliberately models YAML
// mechanics rather than duplicating PipelineConfig's field schema.
type compositionDocument struct {
root *compositionNode
sources []string
}
type compositionNode struct {
kind yaml.Kind
tag string
value string
path string
sources []string
fields []compositionField
items []*compositionNode
}
type compositionField struct {
key string
value *compositionNode
}
// compositionValueRecord is a deterministic semantic leaf projection. Lists
// are atomic configuration values, while mappings are traversed recursively.
type compositionValueRecord struct {
Path string
Kind yaml.Kind
Value string
Sources []string
}
// parseCompositionDocument parses exactly one YAML mapping while retaining
// source ownership, explicit zero values, and declaration order.
func parseCompositionDocument(source string, reader io.Reader) (*compositionDocument, error) {
if strings.TrimSpace(source) == "" {
return nil, fmt.Errorf("configuration source identity is required")
}
if reader == nil {
return nil, fmt.Errorf("configuration source %q: reader is nil", source)
}
decoder := yaml.NewDecoder(reader)
var document yaml.Node
if err := decoder.Decode(&document); err != nil {
if err == io.EOF {
return nil, fmt.Errorf("configuration source %q: document is empty", source)
}
return nil, fmt.Errorf("configuration source %q: decode YAML: %w", source, err)
}
var trailing yaml.Node
if err := decoder.Decode(&trailing); err == nil {
return nil, fmt.Errorf("configuration source %q: must contain exactly one YAML document", source)
} else if err != io.EOF {
return nil, fmt.Errorf("configuration source %q: decode trailing YAML: %w", source, err)
}
if document.Kind != yaml.DocumentNode || len(document.Content) != 1 {
return nil, fmt.Errorf("configuration source %q: must contain exactly one YAML document", source)
}
if document.Content[0].Kind != yaml.MappingNode {
return nil, fmt.Errorf(
"configuration source %q: top-level document must be a mapping, got %s",
source,
yamlKindName(document.Content[0].Kind),
)
}
root, err := buildCompositionNode(document.Content[0], source, "")
if err != nil {
return nil, err
}
return &compositionDocument{root: root, sources: []string{source}}, nil
}
func parseCompositionBytes(source string, data []byte) (*compositionDocument, error) {
return parseCompositionDocument(source, bytes.NewReader(data))
}
func buildCompositionNode(node *yaml.Node, source, path string) (*compositionNode, error) {
if node == nil {
return nil, fmt.Errorf("configuration source %q at %s: YAML node is nil", source, displayCompositionPath(path))
}
if node.Kind == yaml.AliasNode {
return nil, compositionNodeError(source, path, node, "YAML aliases are not supported because source ownership would be ambiguous")
}
result := &compositionNode{
kind: node.Kind, tag: node.Tag, value: node.Value, path: path,
sources: []string{source},
}
switch node.Kind {
case yaml.MappingNode:
if len(node.Content)%2 != 0 {
return nil, compositionNodeError(source, path, node, "mapping has an incomplete key/value pair")
}
seen := make(map[string]*yaml.Node, len(node.Content)/2)
for index := 0; index < len(node.Content); index += 2 {
keyNode := node.Content[index]
valueNode := node.Content[index+1]
if keyNode.Kind == yaml.AliasNode {
return nil, compositionNodeError(source, path, keyNode, "YAML aliases are not supported because source ownership would be ambiguous")
}
if valueNode.Kind == yaml.AliasNode {
return nil, compositionNodeError(source, appendCompositionPath(path, keyNode.Value), valueNode, "YAML aliases are not supported because source ownership would be ambiguous")
}
if keyNode.Kind != yaml.ScalarNode || keyNode.Tag != "!!str" {
return nil, compositionNodeError(source, path, keyNode, "mapping keys must be strings")
}
key := keyNode.Value
fieldPath := appendCompositionPath(path, key)
if prior, duplicate := seen[key]; duplicate {
return nil, fmt.Errorf(
"configuration source %q at %s: duplicate YAML key %q (first declared at line %d, column %d; repeated at line %d, column %d)",
source, displayCompositionPath(fieldPath), key,
prior.Line, prior.Column, keyNode.Line, keyNode.Column,
)
}
seen[key] = keyNode
child, err := buildCompositionNode(valueNode, source, fieldPath)
if err != nil {
return nil, err
}
result.fields = append(result.fields, compositionField{key: key, value: child})
}
case yaml.SequenceNode:
for index, childNode := range node.Content {
childPath := fmt.Sprintf("%s[%d]", path, index)
child, err := buildCompositionNode(childNode, source, childPath)
if err != nil {
return nil, err
}
result.items = append(result.items, child)
}
case yaml.ScalarNode:
// Scalar tag and lexical value retain distinctions such as explicit
// false, zero, an empty string, and null until final strict decoding.
default:
return nil, compositionNodeError(source, path, node, fmt.Sprintf("unsupported YAML node kind %s", yamlKindName(node.Kind)))
}
return result, nil
}
// mergeAdditiveComposition recursively combines disjoint mappings. A scalar,
// list, or final keyed value may have only one base owner, regardless of
// whether duplicate values happen to be equal.
func mergeAdditiveComposition(base, incoming *compositionDocument) (*compositionDocument, error) {
return mergeAdditiveCompositions(base, incoming)
}
// mergeAdditiveCompositions validates the complete base source set before
// merging so a conflict names every source that claims the same final path.
func mergeAdditiveCompositions(documents ...*compositionDocument) (*compositionDocument, error) {
if len(documents) == 0 {
return nil, fmt.Errorf("configuration additive base merge requires at least one document")
}
for index, document := range documents {
if err := validateCompositionDocument(document, fmt.Sprintf("base[%d]", index)); err != nil {
return nil, err
}
}
if err := validateAdditiveClaims(documents); err != nil {
return nil, err
}
result := &compositionDocument{
root: cloneCompositionNode(documents[0].root),
sources: append([]string(nil), documents[0].sources...),
}
for _, incoming := range documents[1:] {
merged, err := mergeAdditiveNodes(result.root, incoming.root)
if err != nil {
return nil, err
}
result.root = merged
result.sources = appendUniqueStrings(result.sources, incoming.sources...)
}
return result, nil
}
func validateAdditiveClaims(documents []*compositionDocument) error {
claims := make(map[string][]*compositionNode)
for _, document := range documents {
appendCompositionClaims(document.root, claims)
}
paths := make([]string, 0, len(claims))
for path := range claims {
paths = append(paths, path)
}
sort.Slice(paths, func(i, j int) bool {
leftDepth := compositionPathDepth(paths[i])
rightDepth := compositionPathDepth(paths[j])
if leftDepth != rightDepth {
return leftDepth < rightDepth
}
return paths[i] < paths[j]
})
for _, path := range paths {
values := claims[path]
if len(values) < 2 {
continue
}
allPopulatedMappings := true
for _, value := range values {
if value.kind != yaml.MappingNode || len(value.fields) == 0 {
allPopulatedMappings = false
break
}
}
if !allPopulatedMappings {
return newCompositionConflict("additive base merge", path, values...)
}
}
return nil
}
func appendCompositionClaims(node *compositionNode, claims map[string][]*compositionNode) {
if node == nil {
return
}
if node.path != "" {
claims[node.path] = append(claims[node.path], node)
}
for _, field := range node.fields {
appendCompositionClaims(field.value, claims)
}
}
func compositionPathDepth(path string) int {
if path == "" {
return 0
}
return strings.Count(path, ".") + strings.Count(path, "[") + 1
}
func mergeAdditiveNodes(base, incoming *compositionNode) (*compositionNode, error) {
if base.kind != yaml.MappingNode || incoming.kind != yaml.MappingNode {
return nil, newCompositionConflict("additive base merge", base.path, base, incoming)
}
base.sources = appendUniqueStrings(base.sources, incoming.sources...)
for _, incomingField := range incoming.fields {
index := compositionFieldIndex(base.fields, incomingField.key)
if index < 0 {
field := cloneCompositionField(incomingField)
base.fields = append(base.fields, field)
continue
}
baseValue := base.fields[index].value
incomingValue := incomingField.value
if baseValue.kind == yaml.MappingNode && incomingValue.kind == yaml.MappingNode {
if len(baseValue.fields) == 0 || len(incomingValue.fields) == 0 {
return nil, newCompositionConflict("additive base merge", incomingValue.path, baseValue, incomingValue)
}
merged, err := mergeAdditiveNodes(baseValue, incomingValue)
if err != nil {
return nil, err
}
base.fields[index].value = merged
continue
}
return nil, newCompositionConflict("additive base merge", incomingValue.path, baseValue, incomingValue)
}
return base, nil
}
// mergeOverlayComposition applies the sole overwrite layer. Mappings merge
// recursively; same-kind scalars and lists replace; null and kind changes are
// rejected.
func mergeOverlayComposition(base, overlay *compositionDocument) (*compositionDocument, error) {
if err := validateCompositionDocument(base, "base"); err != nil {
return nil, err
}
if err := validateCompositionDocument(overlay, "overlay"); err != nil {
return nil, err
}
if null := firstNullCompositionNode(overlay.root); null != nil {
return nil, fmt.Errorf(
"configuration overlay at %s from %s: null cannot delete an effective value",
displayCompositionPath(null.path), formatCompositionSources(null.sources),
)
}
merged, err := mergeOverlayNodes(cloneCompositionNode(base.root), overlay.root)
if err != nil {
return nil, err
}
return &compositionDocument{
root: merged,
sources: appendUniqueStrings(
append([]string(nil), base.sources...), overlay.sources...,
),
}, nil
}
func mergeOverlayNodes(base, overlay *compositionNode) (*compositionNode, error) {
if base.kind != yaml.MappingNode || overlay.kind != yaml.MappingNode {
return nil, newCompositionConflict("profile overlay", base.path, base, overlay)
}
base.sources = appendUniqueStrings(base.sources, overlay.sources...)
for _, overlayField := range overlay.fields {
index := compositionFieldIndex(base.fields, overlayField.key)
if index < 0 {
field := cloneCompositionField(overlayField)
base.fields = append(base.fields, field)
continue
}
baseValue := base.fields[index].value
overlayValue := overlayField.value
if baseValue.kind != overlayValue.kind {
return nil, newCompositionConflict("profile overlay kind change", overlayValue.path, baseValue, overlayValue)
}
if baseValue.kind == yaml.MappingNode {
merged, err := mergeOverlayNodes(baseValue, overlayValue)
if err != nil {
return nil, err
}
base.fields[index].value = merged
continue
}
base.fields[index].value = cloneCompositionNode(overlayValue)
}
return base, nil
}
// canonicalYAML renders the effective mapping with sorted keys and normalized
// presentation while retaining sequence order and scalar YAML types.
func (document *compositionDocument) canonicalYAML() ([]byte, error) {
if err := validateCompositionDocument(document, "document"); err != nil {
return nil, err
}
root, err := compositionYAMLNode(document.root)
if err != nil {
return nil, err
}
var buffer bytes.Buffer
encoder := yaml.NewEncoder(&buffer)
encoder.SetIndent(2)
if err := encoder.Encode(root); err != nil {
return nil, fmt.Errorf("render effective configuration YAML: %w", err)
}
if err := encoder.Close(); err != nil {
return nil, fmt.Errorf("finish effective configuration YAML: %w", err)
}
return buffer.Bytes(), nil
}
// canonicalDigestInput provides a deterministic, formatting-independent byte
// representation for later secret-free effective configuration digesting.
func (document *compositionDocument) canonicalDigestInput() ([]byte, error) {
if err := validateCompositionDocument(document, "document"); err != nil {
return nil, err
}
value, err := canonicalCompositionValue(document.root)
if err != nil {
return nil, err
}
data, err := json.Marshal(value)
if err != nil {
return nil, fmt.Errorf("serialize canonical configuration digest input: %w", err)
}
return append(data, '\n'), nil
}
// semanticRecords returns sorted atomic values for future effective diff and
// source-report projections. A caller receives copies of all source slices.
func (document *compositionDocument) semanticRecords() ([]compositionValueRecord, error) {
if err := validateCompositionDocument(document, "document"); err != nil {
return nil, err
}
var records []compositionValueRecord
if err := appendCompositionRecords(document.root, &records); err != nil {
return nil, err
}
sort.Slice(records, func(i, j int) bool { return records[i].Path < records[j].Path })
return records, nil
}
// compactSemanticRecords returns the same logical atomic paths as
// semanticRecords, but represents values as compact JSON-compatible YAML
// values rather than the typed structural form used for digesting. It is the
// stable human-facing projection for semantic comparisons.
func (document *compositionDocument) compactSemanticRecords() ([]compositionValueRecord, error) {
if err := validateCompositionDocument(document, "document"); err != nil {
return nil, err
}
var records []compositionValueRecord
if err := appendCompactCompositionRecords(document.root, &records); err != nil {
return nil, err
}
sort.Slice(records, func(i, j int) bool { return records[i].Path < records[j].Path })
return records, nil
}
func appendCompositionRecords(node *compositionNode, records *[]compositionValueRecord) error {
if node.kind == yaml.MappingNode && len(node.fields) > 0 {
for _, field := range node.fields {
if err := appendCompositionRecords(field.value, records); err != nil {
return err
}
}
return nil
}
value, err := canonicalCompositionValue(node)
if err != nil {
return err
}
encoded, err := json.Marshal(value)
if err != nil {
return fmt.Errorf("serialize configuration value at %s: %w", displayCompositionPath(node.path), err)
}
*records = append(*records, compositionValueRecord{
Path: node.path, Kind: node.kind, Value: string(encoded),
Sources: append([]string(nil), node.sources...),
})
return nil
}
func appendCompactCompositionRecords(node *compositionNode, records *[]compositionValueRecord) error {
if node.kind == yaml.MappingNode && len(node.fields) > 0 {
for _, field := range node.fields {
if err := appendCompactCompositionRecords(field.value, records); err != nil {
return err
}
}
return nil
}
value, err := compactCompositionValue(node)
if err != nil {
return err
}
encoded, err := json.Marshal(value)
if err != nil {
return fmt.Errorf("serialize configuration value at %s: %w", displayCompositionPath(node.path), err)
}
*records = append(*records, compositionValueRecord{
Path: node.path, Kind: node.kind, Value: string(encoded),
Sources: append([]string(nil), node.sources...),
})
return nil
}
func compactCompositionValue(node *compositionNode) (any, error) {
switch node.kind {
case yaml.MappingNode:
values := make(map[string]any, len(node.fields))
for _, field := range node.fields {
value, err := compactCompositionValue(field.value)
if err != nil {
return nil, err
}
values[field.key] = value
}
return values, nil
case yaml.SequenceNode:
values := make([]any, 0, len(node.items))
for _, item := range node.items {
value, err := compactCompositionValue(item)
if err != nil {
return nil, err
}
values = append(values, value)
}
return values, nil
case yaml.ScalarNode:
return canonicalScalarValue(node)
default:
return nil, fmt.Errorf("configuration at %s has unsupported YAML kind %s", displayCompositionPath(node.path), yamlKindName(node.kind))
}
}
type canonicalCompositionField struct {
Key string `json:"key"`
Value any `json:"value"`
}
type canonicalCompositionNode struct {
Kind string `json:"kind"`
Tag string `json:"tag,omitempty"`
Value any `json:"value,omitempty"`
Fields []canonicalCompositionField `json:"fields,omitempty"`
Items []any `json:"items,omitempty"`
}
func canonicalCompositionValue(node *compositionNode) (any, error) {
switch node.kind {
case yaml.MappingNode:
fields := append([]compositionField(nil), node.fields...)
sort.Slice(fields, func(i, j int) bool { return fields[i].key < fields[j].key })
result := canonicalCompositionNode{Kind: "mapping"}
if len(fields) == 0 {
result.Fields = []canonicalCompositionField{}
}
for _, field := range fields {
value, err := canonicalCompositionValue(field.value)
if err != nil {
return nil, err
}
result.Fields = append(result.Fields, canonicalCompositionField{Key: field.key, Value: value})
}
return result, nil
case yaml.SequenceNode:
result := canonicalCompositionNode{Kind: "sequence", Items: make([]any, 0, len(node.items))}
for _, item := range node.items {
value, err := canonicalCompositionValue(item)
if err != nil {
return nil, err
}
result.Items = append(result.Items, value)
}
return result, nil
case yaml.ScalarNode:
value, err := canonicalScalarValue(node)
if err != nil {
return nil, err
}
return canonicalCompositionNode{Kind: "scalar", Tag: node.tag, Value: value}, nil
default:
return nil, fmt.Errorf("configuration at %s has unsupported YAML kind %s", displayCompositionPath(node.path), yamlKindName(node.kind))
}
}
func canonicalScalarValue(node *compositionNode) (any, error) {
raw := &yaml.Node{Kind: yaml.ScalarNode, Tag: node.tag, Value: node.value}
var value any
if err := raw.Decode(&value); err != nil {
return nil, fmt.Errorf("decode scalar at %s: %w", displayCompositionPath(node.path), err)
}
switch typed := value.(type) {
case nil, bool, string, int, int64, uint64, float64:
return typed, nil
default:
// yaml.v3 may decode timestamps or uncommon scalar tags into types that
// encoding/json can serialize deterministically. Preserve the resolved
// tag alongside the value in the containing canonical node.
return typed, nil
}
}
func compositionYAMLNode(node *compositionNode) (*yaml.Node, error) {
switch node.kind {
case yaml.MappingNode:
result := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
fields := append([]compositionField(nil), node.fields...)
sort.Slice(fields, func(i, j int) bool { return fields[i].key < fields[j].key })
for _, field := range fields {
value, err := compositionYAMLNode(field.value)
if err != nil {
return nil, err
}
result.Content = append(result.Content,
&yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: field.key},
value,
)
}
return result, nil
case yaml.SequenceNode:
result := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
for _, item := range node.items {
value, err := compositionYAMLNode(item)
if err != nil {
return nil, err
}
result.Content = append(result.Content, value)
}
return result, nil
case yaml.ScalarNode:
return normalizedCompositionScalarNode(node)
default:
return nil, fmt.Errorf("configuration at %s has unsupported YAML kind %s", displayCompositionPath(node.path), yamlKindName(node.kind))
}
}
func normalizedCompositionScalarNode(node *compositionNode) (*yaml.Node, error) {
raw := &yaml.Node{Kind: yaml.ScalarNode, Tag: node.tag, Value: node.value}
var value any
if err := raw.Decode(&value); err != nil {
return nil, fmt.Errorf("normalize scalar at %s: %w", displayCompositionPath(node.path), err)
}
normalized := &yaml.Node{}
if err := normalized.Encode(value); err != nil {
return nil, fmt.Errorf("encode normalized scalar at %s: %w", displayCompositionPath(node.path), err)
}
if normalized.Kind != yaml.ScalarNode {
return nil, fmt.Errorf("normalize scalar at %s produced YAML kind %s", displayCompositionPath(node.path), yamlKindName(normalized.Kind))
}
return normalized, nil
}
func validateCompositionDocument(document *compositionDocument, role string) error {
if document == nil || document.root == nil {
return fmt.Errorf("configuration composition %s document is nil", role)
}
if document.root.kind != yaml.MappingNode {
return fmt.Errorf("configuration composition %s root must be a mapping", role)
}
return nil
}
func firstNullCompositionNode(node *compositionNode) *compositionNode {
if node == nil {
return nil
}
if node.kind == yaml.ScalarNode && node.tag == "!!null" {
return node
}
for _, field := range node.fields {
if found := firstNullCompositionNode(field.value); found != nil {
return found
}
}
for _, item := range node.items {
if found := firstNullCompositionNode(item); found != nil {
return found
}
}
return nil
}
func cloneCompositionNode(node *compositionNode) *compositionNode {
if node == nil {
return nil
}
clone := &compositionNode{
kind: node.kind, tag: node.tag, value: node.value, path: node.path,
sources: append([]string(nil), node.sources...),
}
for _, field := range node.fields {
clone.fields = append(clone.fields, cloneCompositionField(field))
}
for _, item := range node.items {
clone.items = append(clone.items, cloneCompositionNode(item))
}
return clone
}
func cloneCompositionField(field compositionField) compositionField {
return compositionField{key: field.key, value: cloneCompositionNode(field.value)}
}
func compositionFieldIndex(fields []compositionField, key string) int {
for index := range fields {
if fields[index].key == key {
return index
}
}
return -1
}
func appendCompositionPath(parent, key string) string {
if isSimpleCompositionPathSegment(key) {
if parent == "" {
return key
}
return parent + "." + key
}
if parent == "" {
return "[" + strconv.Quote(key) + "]"
}
return parent + "[" + strconv.Quote(key) + "]"
}
func isSimpleCompositionPathSegment(value string) bool {
if value == "" {
return false
}
for index, char := range value {
if (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') || char == '_' || (index > 0 && char >= '0' && char <= '9') || (index > 0 && char == '-') {
continue
}
return false
}
return true
}
func displayCompositionPath(path string) string {
if path == "" {
return "<root>"
}
return path
}
func compositionNodeError(source, path string, node *yaml.Node, message string) error {
line, column := 0, 0
if node != nil {
line, column = node.Line, node.Column
}
return fmt.Errorf(
"configuration source %q at %s (line %d, column %d): %s",
source, displayCompositionPath(path), line, column, message,
)
}
func newCompositionConflict(operation, path string, values ...*compositionNode) error {
var sources []string
var kinds []string
for _, value := range values {
if value == nil {
continue
}
sources = appendUniqueStrings(sources, compositionClaimSources(value)...)
kind := yamlKindName(value.kind)
if !containsString(kinds, kind) {
kinds = append(kinds, kind)
}
}
return fmt.Errorf(
"configuration %s conflict at %s: claimed by %s (YAML kinds: %s)",
operation, displayCompositionPath(path), formatCompositionSources(sources), strings.Join(kinds, ", "),
)
}
func compositionClaimSources(node *compositionNode) []string {
if node == nil {
return nil
}
sources := append([]string(nil), node.sources...)
for _, field := range node.fields {
sources = appendUniqueStrings(sources, compositionClaimSources(field.value)...)
}
for _, item := range node.items {
sources = appendUniqueStrings(sources, compositionClaimSources(item)...)
}
return sources
}
func appendUniqueStrings(values []string, additions ...string) []string {
seen := make(map[string]struct{}, len(values)+len(additions))
result := make([]string, 0, len(values)+len(additions))
for _, value := range append(append([]string(nil), values...), additions...) {
if _, exists := seen[value]; exists {
continue
}
seen[value] = struct{}{}
result = append(result, value)
}
return result
}
func containsString(values []string, target string) bool {
for _, value := range values {
if value == target {
return true
}
}
return false
}
func formatCompositionSources(sources []string) string {
quoted := make([]string, 0, len(sources))
for _, source := range sources {
quoted = append(quoted, strconv.Quote(source))
}
return strings.Join(quoted, ", ")
}
func yamlKindName(kind yaml.Kind) string {
switch kind {
case yaml.DocumentNode:
return "document"
case yaml.MappingNode:
return "mapping"
case yaml.SequenceNode:
return "sequence"
case yaml.ScalarNode:
return "scalar"
case yaml.AliasNode:
return "alias"
default:
return fmt.Sprintf("kind(%d)", kind)
}
}

View File

@@ -1,353 +0,0 @@
package config
import (
"reflect"
"strings"
"testing"
"gopkg.in/yaml.v3"
)
func TestParseCompositionDocumentRetainsPresenceOwnershipAndDeclarationOrder(t *testing.T) {
document := mustParseComposition(t, "root.yml", `zeta: false
zero: 0
empty_map: {}
empty_list: []
nested:
value: ""
`)
wantOrder := []string{"zeta", "zero", "empty_map", "empty_list", "nested"}
gotOrder := make([]string, 0, len(document.root.fields))
for _, field := range document.root.fields {
gotOrder = append(gotOrder, field.key)
}
if !reflect.DeepEqual(gotOrder, wantOrder) {
t.Fatalf("declaration order = %#v, want %#v", gotOrder, wantOrder)
}
tests := []struct {
path string
kind yaml.Kind
tag string
value string
}{
{path: "zeta", kind: yaml.ScalarNode, tag: "!!bool", value: "false"},
{path: "zero", kind: yaml.ScalarNode, tag: "!!int", value: "0"},
{path: "empty_map", kind: yaml.MappingNode, tag: "!!map"},
{path: "empty_list", kind: yaml.SequenceNode, tag: "!!seq"},
{path: "nested.value", kind: yaml.ScalarNode, tag: "!!str", value: ""},
}
for _, tt := range tests {
node := compositionNodeAtPath(t, document.root, tt.path)
if node.kind != tt.kind || node.tag != tt.tag || node.value != tt.value || !reflect.DeepEqual(node.sources, []string{"root.yml"}) {
t.Fatalf("node %s = kind=%v tag=%q value=%q sources=%#v", tt.path, node.kind, node.tag, node.value, node.sources)
}
}
}
func TestParseCompositionDocumentRejectsAmbiguousOrMalformedYAML(t *testing.T) {
tests := []struct {
name string
yaml string
want string
}{
{name: "duplicate top-level key", yaml: "value: 1\nvalue: 2\n", want: "duplicate YAML key"},
{name: "duplicate nested key", yaml: "outer:\n value: 1\n value: 2\n", want: "outer.value"},
{name: "alias", yaml: "base: &base\n value: 1\ncopy: *base\n", want: "aliases are not supported"},
{name: "trailing document", yaml: "value: 1\n---\nvalue: 2\n", want: "exactly one YAML document"},
{name: "top-level sequence", yaml: "- value\n", want: "top-level document must be a mapping"},
{name: "non-string key", yaml: "1: value\n", want: "mapping keys must be strings"},
{name: "malformed", yaml: "outer: [\n", want: "decode YAML"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := parseCompositionDocument("broken.yml", strings.NewReader(tt.yaml))
if err == nil {
t.Fatal("parseCompositionDocument() error = nil")
}
if !strings.Contains(err.Error(), "broken.yml") || !strings.Contains(err.Error(), tt.want) {
t.Fatalf("error = %q, want source and %q", err, tt.want)
}
})
}
}
func TestMergeAdditiveCompositionJoinsOnlyDisjointMappings(t *testing.T) {
root := mustParseComposition(t, "pipeline.yml", `scriptorium:
artifacts:
recap:
enabled: true
zero: 0
`)
imports := mustParseComposition(t, "conf.d/artifacts.yml", `scriptorium:
artifacts:
handout:
enabled: false
empty_list: []
`)
merged, err := mergeAdditiveComposition(root, imports)
if err != nil {
t.Fatalf("mergeAdditiveComposition() error = %v", err)
}
for _, path := range []string{
"empty_list", "scriptorium.artifacts.handout.enabled",
"scriptorium.artifacts.recap.enabled", "zero",
} {
_ = compositionNodeAtPath(t, merged.root, path)
}
if got := compositionNodeAtPath(t, merged.root, "scriptorium.artifacts.handout.enabled").sources; !reflect.DeepEqual(got, []string{"conf.d/artifacts.yml"}) {
t.Fatalf("handout sources = %#v", got)
}
if got := compositionNodeAtPath(t, merged.root, "scriptorium.artifacts.recap.enabled").sources; !reflect.DeepEqual(got, []string{"pipeline.yml"}) {
t.Fatalf("recap sources = %#v", got)
}
// Merge operations return a new document and retain the declared order in
// each input for source-aware diagnostics.
if len(root.root.fields) != 2 || len(imports.root.fields) != 2 {
t.Fatalf("merge mutated inputs: root=%d import=%d", len(root.root.fields), len(imports.root.fields))
}
}
func TestMergeAdditiveCompositionRejectsEveryDuplicateClass(t *testing.T) {
tests := []struct {
name string
baseYAML string
nextYAML string
path string
}{
{name: "equal scalar", baseYAML: "value: true\n", nextYAML: "value: true\n", path: "value"},
{name: "different scalar", baseYAML: "value: true\n", nextYAML: "value: false\n", path: "value"},
{name: "atomic list", baseYAML: "values: [one]\n", nextYAML: "values: [two]\n", path: "values"},
{name: "keyed entry", baseYAML: "items:\n shared:\n left: 1\n", nextYAML: "items:\n shared:\n left: 2\n", path: "items.shared.left"},
{name: "kind conflict", baseYAML: "value:\n nested: true\n", nextYAML: "value: scalar\n", path: "value"},
{name: "duplicate empty map", baseYAML: "value: {}\n", nextYAML: "value: {}\n", path: "value"},
{name: "empty map then populated map", baseYAML: "value: {}\n", nextYAML: "value: {nested: true}\n", path: "value"},
{name: "populated map then empty map", baseYAML: "value: {nested: true}\n", nextYAML: "value: {}\n", path: "value"},
{name: "duplicate empty list", baseYAML: "value: []\n", nextYAML: "value: []\n", path: "value"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
base := mustParseComposition(t, "base.yml", tt.baseYAML)
next := mustParseComposition(t, "next.yml", tt.nextYAML)
_, err := mergeAdditiveComposition(base, next)
if err == nil {
t.Fatal("mergeAdditiveComposition() error = nil")
}
for _, want := range []string{tt.path, "base.yml", "next.yml"} {
if !strings.Contains(err.Error(), want) {
t.Fatalf("error = %q, want %q", err, want)
}
}
})
}
}
func TestMergeAdditiveCompositionReportsAllClaimingSources(t *testing.T) {
left := mustParseComposition(t, "left.yml", "group:\n left: 1\n")
right := mustParseComposition(t, "right.yml", "group:\n right: 2\n")
base, err := mergeAdditiveComposition(left, right)
if err != nil {
t.Fatal(err)
}
overlap := mustParseComposition(t, "overlap.yml", "group: scalar\n")
_, err = mergeAdditiveComposition(base, overlap)
if err == nil {
t.Fatal("mergeAdditiveComposition() error = nil")
}
for _, want := range []string{"group", "left.yml", "right.yml", "overlap.yml"} {
if !strings.Contains(err.Error(), want) {
t.Fatalf("error = %q, want %q", err, want)
}
}
_, err = mergeAdditiveCompositions(
mustParseComposition(t, "first.yml", "value: 1\n"),
mustParseComposition(t, "second.yml", "value: 2\n"),
mustParseComposition(t, "third.yml", "value: 3\n"),
)
if err == nil {
t.Fatal("mergeAdditiveCompositions() error = nil")
}
for _, want := range []string{"value", "first.yml", "second.yml", "third.yml"} {
if !strings.Contains(err.Error(), want) {
t.Fatalf("error = %q, want %q", err, want)
}
}
}
func TestMergeOverlayCompositionRecursesMapsAndReplacesAtomicValues(t *testing.T) {
base := mustParseComposition(t, "base.yml", `feature:
enabled: true
retries: 3
values: [one, two]
inherited: kept
artifacts:
recap:
enabled: true
`)
overlay := mustParseComposition(t, "testing.yml", `feature:
enabled: false
retries: 0
values: []
added: present
artifacts:
handout:
enabled: false
`)
merged, err := mergeOverlayComposition(base, overlay)
if err != nil {
t.Fatalf("mergeOverlayComposition() error = %v", err)
}
assertCompositionScalar(t, merged.root, "feature.enabled", "!!bool", "false", "testing.yml")
assertCompositionScalar(t, merged.root, "feature.retries", "!!int", "0", "testing.yml")
assertCompositionScalar(t, merged.root, "feature.inherited", "!!str", "kept", "base.yml")
assertCompositionScalar(t, merged.root, "feature.added", "!!str", "present", "testing.yml")
if values := compositionNodeAtPath(t, merged.root, "feature.values"); values.kind != yaml.SequenceNode || len(values.items) != 0 || !reflect.DeepEqual(values.sources, []string{"testing.yml"}) {
t.Fatalf("replaced list = %#v", values)
}
_ = compositionNodeAtPath(t, merged.root, "artifacts.recap.enabled")
_ = compositionNodeAtPath(t, merged.root, "artifacts.handout.enabled")
}
func TestMergeOverlayCompositionRejectsKindChangesAndNullDeletion(t *testing.T) {
kindTests := []struct {
name string
baseYAML string
overlay string
}{
{name: "map to scalar", baseYAML: "value: {nested: true}\n", overlay: "value: replacement\n"},
{name: "scalar to map", baseYAML: "value: original\n", overlay: "value: {nested: true}\n"},
{name: "list to scalar", baseYAML: "value: [one]\n", overlay: "value: replacement\n"},
{name: "scalar to list", baseYAML: "value: original\n", overlay: "value: [one]\n"},
}
for _, tt := range kindTests {
t.Run(tt.name, func(t *testing.T) {
_, err := mergeOverlayComposition(
mustParseComposition(t, "base.yml", tt.baseYAML),
mustParseComposition(t, "overlay.yml", tt.overlay),
)
if err == nil || !strings.Contains(err.Error(), "value") || !strings.Contains(err.Error(), "kind change") {
t.Fatalf("error = %v, want value kind change", err)
}
})
}
for _, overlayYAML := range []string{"value: null\n", "value: ~\n", "nested:\n value:\n"} {
_, err := mergeOverlayComposition(
mustParseComposition(t, "base.yml", "value: original\nnested:\n value: original\n"),
mustParseComposition(t, "overlay.yml", overlayYAML),
)
if err == nil || !strings.Contains(err.Error(), "null cannot delete") || !strings.Contains(err.Error(), "overlay.yml") {
t.Fatalf("error = %v, want source-qualified null rejection", err)
}
}
}
func TestCompositionCanonicalOutputsAreDeterministic(t *testing.T) {
first := mustParseComposition(t, "first.yml", `zeta: 01
alpha:
list: [true, false]
empty: {}
`)
second := mustParseComposition(t, "second.yml", `alpha:
empty: {}
list:
- true
- false
zeta: 1
`)
firstYAML, err := first.canonicalYAML()
if err != nil {
t.Fatal(err)
}
secondYAML, err := second.canonicalYAML()
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(firstYAML, secondYAML) {
t.Fatalf("canonical YAML differs:\n%s\n---\n%s", firstYAML, secondYAML)
}
firstDigest, err := first.canonicalDigestInput()
if err != nil {
t.Fatal(err)
}
secondDigest, err := second.canonicalDigestInput()
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(firstDigest, secondDigest) {
t.Fatalf("digest input differs:\n%s\n---\n%s", firstDigest, secondDigest)
}
left := mustParseComposition(t, "left.yml", "zeta: 1\n")
right := mustParseComposition(t, "right.yml", "alpha: 2\n")
leftRight, err := mergeAdditiveComposition(left, right)
if err != nil {
t.Fatal(err)
}
rightLeft, err := mergeAdditiveComposition(right, left)
if err != nil {
t.Fatal(err)
}
want, _ := leftRight.canonicalDigestInput()
got, _ := rightLeft.canonicalDigestInput()
if !reflect.DeepEqual(got, want) {
t.Fatalf("source traversal changed semantic digest input: got %s want %s", got, want)
}
wantYAML, _ := leftRight.canonicalYAML()
gotYAML, _ := rightLeft.canonicalYAML()
if !reflect.DeepEqual(gotYAML, wantYAML) {
t.Fatalf("source traversal changed canonical YAML: got %s want %s", gotYAML, wantYAML)
}
records, err := first.semanticRecords()
if err != nil {
t.Fatal(err)
}
paths := make([]string, 0, len(records))
for _, record := range records {
paths = append(paths, record.Path)
}
if wantPaths := []string{"alpha.empty", "alpha.list", "zeta"}; !reflect.DeepEqual(paths, wantPaths) {
t.Fatalf("semantic record paths = %#v, want %#v", paths, wantPaths)
}
}
func mustParseComposition(t *testing.T, source, input string) *compositionDocument {
t.Helper()
document, err := parseCompositionBytes(source, []byte(input))
if err != nil {
t.Fatalf("parseCompositionBytes(%q) error = %v", source, err)
}
return document
}
func compositionNodeAtPath(t *testing.T, root *compositionNode, path string) *compositionNode {
t.Helper()
node := root
for _, segment := range strings.Split(path, ".") {
if node == nil || node.kind != yaml.MappingNode {
t.Fatalf("path %q reached non-mapping at %q", path, segment)
}
index := compositionFieldIndex(node.fields, segment)
if index < 0 {
t.Fatalf("path %q missing segment %q", path, segment)
}
node = node.fields[index].value
}
return node
}
func assertCompositionScalar(t *testing.T, root *compositionNode, path, tag, value, source string) {
t.Helper()
node := compositionNodeAtPath(t, root, path)
if node.kind != yaml.ScalarNode || node.tag != tag || node.value != value || !reflect.DeepEqual(node.sources, []string{source}) {
t.Fatalf("%s = kind=%s tag=%q value=%q sources=%#v", path, yamlKindName(node.kind), node.tag, node.value, node.sources)
}
}

View File

@@ -191,8 +191,6 @@ type NormalizeConfig struct {
OutputPath string `yaml:"output_path" json:"output_path"`
OutputSchema string `yaml:"output_schema" json:"output_schema"`
Report *bool `yaml:"report" json:"report"`
outputPathSet bool `yaml:"-" json:"-"`
}
// TrimConfig configures trim-stage transcript boundary behavior.

View File

@@ -74,7 +74,7 @@ func finalizePipelineResolution(cfg *PipelineConfig) error {
// recomputePipelineEffectiveDigest is the single package-owned hook for
// refreshing provenance after later resolution expands concrete pipeline
// values. Composition declarations and runtime provenance are not serialized.
// values. Loader directives and runtime provenance are not serialized.
func recomputePipelineEffectiveDigest(cfg *PipelineConfig) error {
if cfg == nil || cfg.resolution == nil {
return fmt.Errorf("pipeline resolution metadata is required")

View File

@@ -55,7 +55,7 @@ type EffectivePipelineValueRecord struct {
// EffectivePipelineValues projects a fully resolved pipeline into sorted
// logical configuration values. Mappings are flattened, while sequences remain
// atomic values. The projection uses the same normalized effective mapping as
// config show and therefore excludes composition and family declarations.
// config show and therefore excludes loader directives and family declarations.
func EffectivePipelineValues(cfg *PipelineConfig) ([]EffectivePipelineValueRecord, error) {
if cfg == nil {
return nil, fmt.Errorf("pipeline config is required")
@@ -301,6 +301,9 @@ func pipelineSourceRole(cfg *PipelineConfig, source string) string {
if source == pipelineDefaultOwnershipSource {
return "default"
}
if source == "resolution" {
return "resolution"
}
if cfg == nil || cfg.resolution == nil {
return "pipeline"
}
@@ -319,7 +322,7 @@ func pipelineSourceRole(cfg *PipelineConfig, source string) string {
}
func normalizedProvenanceSource(source string) string {
if source == pipelineDefaultOwnershipSource {
if source == pipelineDefaultOwnershipSource || source == "resolution" {
return source
}
return normalizedProvenancePath(source)

View File

@@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"sort"
"strings"
)
// effectivePipelineProjection is the single safe, schema-specific view used
@@ -37,6 +38,11 @@ type effectiveScriptoriumConfig struct {
Artifacts map[string]ScriptoriumArtifactConfig `yaml:"artifacts" json:"artifacts"`
}
type projectedPipelineValue struct {
record EffectivePipelineValueRecord
segments []string
}
func projectEffectivePipeline(cfg *PipelineConfig, logicalNotarius bool) effectivePipelineProjection {
projection := effectivePipelineProjection{
Workspace: cfg.Workspace, Campaigns: cfg.Campaigns, Storage: cfg.Storage,
@@ -70,23 +76,35 @@ func projectionJSON(projection effectivePipelineProjection) ([]byte, error) {
}
func projectionValueRecords(projection effectivePipelineProjection) ([]EffectivePipelineValueRecord, error) {
data, err := projectionJSON(projection)
detailed, err := projectionDetailedRecords(projection)
if err != nil {
return nil, err
}
records := make([]EffectivePipelineValueRecord, 0, len(detailed))
for _, value := range detailed {
records = append(records, value.record)
}
return records, nil
}
func projectionDetailedRecords(value any) ([]projectedPipelineValue, error) {
data, err := json.Marshal(value)
if err != nil {
return nil, fmt.Errorf("encode typed pipeline projection: %w", err)
}
decoder := json.NewDecoder(bytes.NewReader(data))
decoder.UseNumber()
var root map[string]any
if err := decoder.Decode(&root); err != nil {
return nil, fmt.Errorf("decode effective pipeline projection: %w", err)
}
var records []EffectivePipelineValueRecord
appendProjectionRecords(&records, "", root)
sort.Slice(records, func(i, j int) bool { return records[i].Path < records[j].Path })
var records []projectedPipelineValue
appendProjectionRecords(&records, nil, root)
sort.Slice(records, func(i, j int) bool { return records[i].record.Path < records[j].record.Path })
return records, nil
}
func appendProjectionRecords(records *[]EffectivePipelineValueRecord, path string, value any) {
func appendProjectionRecords(records *[]projectedPipelineValue, segments []string, value any) {
if mapping, ok := value.(map[string]any); ok && len(mapping) > 0 {
keys := make([]string, 0, len(mapping))
for key := range mapping {
@@ -94,14 +112,15 @@ func appendProjectionRecords(records *[]EffectivePipelineValueRecord, path strin
}
sort.Strings(keys)
for _, key := range keys {
childPath := key
if path != "" {
childPath = path + "." + key
}
appendProjectionRecords(records, childPath, mapping[key])
child := append(append([]string(nil), segments...), key)
appendProjectionRecords(records, child, mapping[key])
}
return
}
encoded, _ := json.Marshal(value)
*records = append(*records, EffectivePipelineValueRecord{Path: path, Value: string(encoded)})
path := strings.Join(segments, ".")
*records = append(*records, projectedPipelineValue{
record: EffectivePipelineValueRecord{Path: path, Value: string(encoded)},
segments: append([]string(nil), segments...),
})
}

View File

@@ -17,9 +17,8 @@ func LoadPipeline(path string) (*PipelineConfig, error) {
return LoadPipelineWithOptions(path, PipelineLoadOptions{})
}
// PipelineLoadOptions carries an optional explicit profile selection. A nil
// Profile means the caller omitted selection; a non-nil empty value is an
// explicit invalid selection.
// PipelineLoadOptions carries an optional explicit profile selection. Nil,
// empty, and the exact name "default" select the base configuration.
type PipelineLoadOptions struct {
Profile *string
}
@@ -27,77 +26,32 @@ type PipelineLoadOptions struct {
// LoadPipelineWithOptions loads pipeline configuration with strict field
// checking and optional named-profile selection.
func LoadPipelineWithOptions(path string, opts PipelineLoadOptions) (*PipelineConfig, error) {
sources, err := loadPipelineCompositionSources(path)
cfg, err := loadPipelineWithConfigKit(path, opts)
if err != nil {
return nil, fmt.Errorf("load pipeline config: %w", err)
}
if _, err := selectPipelineProfile(sources.envelope, opts); err != nil {
return nil, fmt.Errorf("load pipeline config: %w", err)
}
if err := sources.loadOverlays(); err != nil {
return nil, fmt.Errorf("load pipeline config: %w", err)
}
cfg, err := sources.resolve(opts)
if err != nil {
return nil, fmt.Errorf("load pipeline config: %w", err)
return nil, err
}
return finalizeLoadedPipeline(path, cfg)
}
// LoadPipelineProfilePair resolves two explicit named profiles from one parsed
// pipeline root and its declared source set. Each result is independently
// decoded, defaulted, and finalized so later resolution can safely mutate one
// effective pipeline without affecting the other.
// LoadPipelineProfilePair resolves two profiles as independent ConfigKit loads.
func LoadPipelineProfilePair(path, leftProfile, rightProfile string) (*PipelineConfig, *PipelineConfig, error) {
if _, err := normalizePipelineProfileName(leftProfile, "left profile selection"); err != nil {
return nil, nil, fmt.Errorf("load left pipeline profile: %w", err)
}
if _, err := normalizePipelineProfileName(rightProfile, "right profile selection"); err != nil {
return nil, nil, fmt.Errorf("load right pipeline profile: %w", err)
}
if leftProfile == rightProfile {
return nil, nil, fmt.Errorf("load pipeline profiles: left and right profile selections must differ")
}
sources, err := loadPipelineCompositionSources(path)
if err != nil {
return nil, nil, fmt.Errorf("load pipeline config: %w", err)
}
leftOptions := PipelineLoadOptions{Profile: &leftProfile}
rightOptions := PipelineLoadOptions{Profile: &rightProfile}
if _, err := selectPipelineProfile(sources.envelope, leftOptions); err != nil {
return nil, nil, fmt.Errorf("load left pipeline profile: %w", err)
}
if _, err := selectPipelineProfile(sources.envelope, rightOptions); err != nil {
return nil, nil, fmt.Errorf("load right pipeline profile: %w", err)
}
if err := sources.loadOverlays(); err != nil {
return nil, nil, fmt.Errorf("load pipeline config: %w", err)
}
left, err := sources.resolve(leftOptions)
left, err := LoadPipelineWithOptions(path, leftOptions)
if err != nil {
return nil, nil, fmt.Errorf("load left pipeline profile: %w", err)
}
right, err := sources.resolve(rightOptions)
right, err := LoadPipelineWithOptions(path, rightOptions)
if err != nil {
return nil, nil, fmt.Errorf("load right pipeline profile: %w", err)
}
left, err = finalizeLoadedPipeline(path, left)
if err != nil {
return nil, nil, err
}
right, err = finalizeLoadedPipeline(path, right)
if err != nil {
return nil, nil, err
}
return left, right, nil
}
func finalizeLoadedPipeline(path string, cfg *PipelineConfig) (*PipelineConfig, error) {
cfg.resolution.publishDeclared = cfg.Publish != nil
applyPipelineDefaults(cfg)
if err := NormalizePipelineConfig(cfg); err != nil {
return nil, fmt.Errorf("load pipeline config: %w", err)
}
captureLogicalNotariusPaths(cfg)
if err := resolveNotariusPaths(cfg, path); err != nil {
return nil, fmt.Errorf("load pipeline config: %w", err)
@@ -504,217 +458,6 @@ func shortName(path, fallback string) string {
return base
}
func applyPipelineDefaults(cfg *PipelineConfig) {
if cfg == nil {
return
}
applyWorkspaceDefaults(&cfg.Workspace)
applyCampaignsDefaults(&cfg.Campaigns)
applyStorageDefaults(&cfg.Storage)
applySpoolDefaults(&cfg.Spool)
applyCacheDefaults(&cfg.Cache)
applyPublishDefaults(&cfg.Publish)
applyWhisperXDefaults(&cfg.WhisperX)
applySeriatimDefaults(&cfg.Seriatim)
applyAuditaDefaults(&cfg.Audita)
if cfg.Normalize == nil {
cfg.Normalize = &NormalizeConfig{}
}
applyNormalizeDefaults(cfg.Normalize)
if cfg.Trim == nil {
cfg.Trim = &TrimConfig{}
}
applyTrimDefaults(cfg.Trim)
if trimEnabled(cfg.Trim) && cfg.Scriptorium == nil {
cfg.Scriptorium = &ScriptoriumConfig{}
}
applyRenderDefaults(&cfg.Render)
applyScriptoriumDefaults(cfg.Scriptorium)
applyNotariusDefaults(cfg.Notarius)
applyNotificationDefaults(&cfg.Notification)
}
func applyNotificationDefaults(cfg *NotificationConfig) {
if cfg == nil {
return
}
if strings.TrimSpace(cfg.Mode) == "" {
cfg.Mode = DefaultNotificationMode
}
}
func applyCampaignsDefaults(cfg *CampaignsConfig) {
if cfg == nil {
return
}
if cfg.Root == "" {
cfg.Root = DefaultCampaignsRoot
}
}
func applyWorkspaceDefaults(cfg *WorkspaceConfig) {
if cfg == nil {
return
}
if cfg.Root == "" {
cfg.Root = DefaultWorkspaceRoot
}
}
func applyStorageDefaults(cfg *StorageConfig) {
if cfg == nil {
return
}
backend := strings.ToLower(strings.TrimSpace(cfg.Backend))
if backend == "" {
backend = StorageBackendLocal
}
cfg.Backend = backend
if cfg.S3 == nil {
return
}
if cfg.S3.RootPrefix == "" {
cfg.S3.RootPrefix = DefaultStorageS3RootPrefix
}
if cfg.S3.AccessKeyIDEnv == "" {
cfg.S3.AccessKeyIDEnv = DefaultS3AccessKeyIDEnv
}
if cfg.S3.SecretKeyEnv == "" {
cfg.S3.SecretKeyEnv = DefaultS3SecretAccessKeyEnv
}
}
func applySpoolDefaults(cfg *SpoolConfig) {
if cfg == nil {
return
}
if cfg.Root == "" {
cfg.Root = DefaultSpoolRoot
}
}
func applyCacheDefaults(cfg *CacheConfig) {
if cfg == nil {
return
}
if cfg.Root == "" {
cfg.Root = DefaultCacheRoot
}
if cfg.S3Audio == nil {
cfg.S3Audio = boolPtr(DefaultCacheS3Audio)
}
}
func applyPublishDefaults(cfg **PublishConfig) {
if cfg == nil {
return
}
if *cfg == nil {
*cfg = &PublishConfig{}
}
if (*cfg).Enabled == nil {
(*cfg).Enabled = boolPtr(DefaultArchiveEnabled)
}
if (*cfg).UploadRun == nil {
(*cfg).UploadRun = boolPtr(DefaultArchiveUploadRun)
}
if len((*cfg).Outputs) == 0 {
(*cfg).Outputs = append([]PublishOutputRule(nil), DefaultPublishOutputs...)
}
for i := range (*cfg).Outputs {
if (*cfg).Outputs[i].Required == nil {
(*cfg).Outputs[i].Required = boolPtr(true)
}
}
}
func applyWhisperXDefaults(cfg *WhisperXConfig) {
if cfg == nil {
return
}
if cfg.Language == "" {
cfg.Language = DefaultWhisperXLanguage
}
if cfg.Timeout == "" {
cfg.Timeout = DefaultWhisperXTimeout
}
if cfg.RetryDelay == "" {
cfg.RetryDelay = DefaultWhisperXRetryDelay
}
if cfg.Concurrency == nil {
cfg.Concurrency = intPtr(DefaultWhisperXConcurrency)
}
if cfg.Retries == nil {
cfg.Retries = intPtr(DefaultWhisperXRetries)
}
}
func intPtr(v int) *int {
p := v
return &p
}
func applySeriatimDefaults(cfg *SeriatimConfig) {
if cfg == nil {
return
}
if cfg.Binary == "" {
cfg.Binary = DefaultSeriatimBinary
}
if cfg.Timeout == "" {
cfg.Timeout = DefaultSeriatimTimeout
}
if cfg.OutputSchema == "" {
cfg.OutputSchema = DefaultSeriatimOutputSchema
}
if cfg.CoalesceGap == nil {
cfg.CoalesceGap = float64Ptr(DefaultSeriatimCoalesceGap)
}
if cfg.Report == nil {
cfg.Report = boolPtr(DefaultSeriatimReport)
}
}
func applyAuditaDefaults(cfg *AuditaConfig) {
if cfg == nil {
return
}
if cfg.Binary == "" {
cfg.Binary = DefaultAuditaBinary
}
if cfg.Timeout == "" {
cfg.Timeout = DefaultAuditaTimeout
}
if cfg.Report == nil {
cfg.Report = boolPtr(DefaultAuditaReport)
}
}
func applyScriptoriumDefaults(cfg *ScriptoriumConfig) {
if cfg == nil {
return
}
if cfg.Binary == "" {
cfg.Binary = DefaultScriptoriumBinary
}
if cfg.Timeout == "" {
cfg.Timeout = DefaultScriptoriumTimeout
}
}
func applyNotariusDefaults(cfg *NotariusConfig) {
if cfg == nil {
return
}
if cfg.Binary == "" {
cfg.Binary = DefaultNotariusBinary
}
if cfg.Timeout == "" {
cfg.Timeout = DefaultNotariusTimeout
}
}
func resolveNotariusPaths(cfg *PipelineConfig, pipelinePath string) error {
if cfg == nil || cfg.Notarius == nil || !cfg.Notarius.Enabled {
return nil
@@ -753,83 +496,3 @@ func resolveNotariusPaths(cfg *PipelineConfig, pipelinePath string) error {
cfg.Notarius.WorkingDirectory = workingDirectory
return nil
}
func applyTrimDefaults(cfg *TrimConfig) {
if cfg == nil {
return
}
if cfg.Enabled == nil {
cfg.Enabled = boolPtr(DefaultTrimEnabled)
}
if strings.TrimSpace(cfg.OutputPath) == "" {
cfg.OutputPath = DefaultTrimOutputPath
}
if strings.TrimSpace(cfg.Bounds.PromptID) == "" {
cfg.Bounds.PromptID = DefaultTrimBoundsPromptID
}
if strings.TrimSpace(cfg.Bounds.TranscriptInputName) == "" {
cfg.Bounds.TranscriptInputName = DefaultTrimBoundsTranscriptInputName
}
if strings.TrimSpace(cfg.Bounds.OutputPath) == "" {
cfg.Bounds.OutputPath = DefaultTrimBoundsOutputPath
}
if cfg.Bounds.Timeout == "" {
cfg.Bounds.Timeout = DefaultTrimBoundsTimeout
}
if cfg.Seriatim.Report == nil {
cfg.Seriatim.Report = boolPtr(DefaultTrimSeriatimReport)
}
}
func trimEnabled(cfg *TrimConfig) bool {
return cfg != nil && cfg.Enabled != nil && *cfg.Enabled
}
func applyRenderDefaults(cfg **RenderConfig) {
if cfg == nil {
return
}
if *cfg == nil {
*cfg = &RenderConfig{}
}
if (*cfg).Enabled == nil {
(*cfg).Enabled = boolPtr(DefaultRenderEnabled)
}
if strings.TrimSpace((*cfg).Format) == "" {
(*cfg).Format = DefaultRenderFormat
}
if strings.TrimSpace((*cfg).Title) == "" {
(*cfg).Title = DefaultRenderTitle
}
if (*cfg).IncludeTimestamps == nil {
(*cfg).IncludeTimestamps = boolPtr(DefaultRenderTimestamps)
}
if (*cfg).IncludeSegmentIDs == nil {
(*cfg).IncludeSegmentIDs = boolPtr(DefaultRenderSegmentIDs)
}
}
func applyNormalizeDefaults(cfg *NormalizeConfig) {
if cfg == nil {
return
}
if cfg.OutputSchema == "" {
cfg.OutputSchema = DefaultNormalizeOutputSchema
}
if cfg.OutputPath == "" && !cfg.outputPathWasSet() {
cfg.OutputPath = DefaultNormalizeOutputPath
}
if cfg.Report == nil {
cfg.Report = boolPtr(DefaultNormalizeReport)
}
}
func float64Ptr(v float64) *float64 {
p := v
return &p
}
func boolPtr(v bool) *bool {
p := v
return &p
}

View File

@@ -48,7 +48,7 @@ func TestLoadersRejectTrailingYAMLDocuments(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := tt.load()
if err == nil || !strings.Contains(err.Error(), "exactly one YAML document") {
if err == nil || (!strings.Contains(err.Error(), "exactly one YAML document") && !strings.Contains(err.Error(), "yaml.multiple_documents")) {
t.Fatalf("load error = %v, want exactly-one-document error", err)
}
})

View File

@@ -1,49 +0,0 @@
package config
import (
"fmt"
"gopkg.in/yaml.v3"
)
// UnmarshalYAML tracks explicit normalize.output_path presence so validation can
// distinguish omitted vs explicitly empty values.
func (cfg *NormalizeConfig) UnmarshalYAML(node *yaml.Node) error {
if node.Kind != yaml.MappingNode {
return fmt.Errorf("normalize must be a mapping")
}
allowedKeys := map[string]struct{}{
"output_path": {},
"output_schema": {},
"report": {},
}
for i := 0; i+1 < len(node.Content); i += 2 {
key := node.Content[i].Value
if _, ok := allowedKeys[key]; !ok {
return fmt.Errorf("field %q not found in type config.NormalizeConfig", key)
}
}
type rawNormalize NormalizeConfig
var raw rawNormalize
if err := node.Decode(&raw); err != nil {
return err
}
*cfg = NormalizeConfig(raw)
for i := 0; i+1 < len(node.Content); i += 2 {
if node.Content[i].Value == "output_path" {
cfg.outputPathSet = true
break
}
}
return nil
}
// outputPathSet records whether normalize.output_path appeared in YAML.
// This is intentionally unexported and not serialized.
func (cfg *NormalizeConfig) outputPathWasSet() bool {
if cfg == nil {
return false
}
return cfg.outputPathSet
}

View File

@@ -107,6 +107,12 @@ notarius:
enabled: true
config_path: notarius/config.yml
pipeline_id: dnd-session
outputs:
npc_registry:
lane_id: npc-registry
media_type: application/json
schema_id: notarius.dnd.npc_registry
schema_version: v1
`
paths := make([]string, 2)
configs := make([]*PipelineConfig, 2)
@@ -151,7 +157,7 @@ func TestNotariusStrictYAML(t *testing.T) {
_, err := LoadPipeline(path)
want := "strict decode failed"
if tt.name == "duplicate reference selector" {
want = "duplicate YAML key"
want = "yaml.duplicate_key"
}
if err == nil || !strings.Contains(err.Error(), want) {
t.Fatalf("LoadPipeline() error = %v, want containing %q", err, want)

View File

@@ -23,12 +23,12 @@ func TestNotificationConfigSupportsOnlyNoopMode(t *testing.T) {
{
name: "backend is rejected",
section: "notification:\n backend: email\n",
wantLoad: "field backend not found",
wantLoad: "decode.unknown_field",
},
{
name: "recipient is rejected",
section: "notification:\n recipient: party@example.com\n",
wantLoad: "field recipient not found",
wantLoad: "decode.unknown_field",
},
{
name: "provider mode is rejected",

View File

@@ -1,532 +0,0 @@
package config
import (
"fmt"
"os"
"path/filepath"
"strings"
"unicode"
"gitea.maximumdirect.net/eric/narratio/internal/pathsafe"
"gopkg.in/yaml.v3"
)
type pipelineResolutionMetadata struct {
rootPath string
imports []string
sources []string
selectedProfile *pipelineProfileSelection
effectiveDigest string
logicalNotariusConfig string
logicalNotariusWorking string
logicalNotariusCaptured bool
ownership []pipelineFieldOwnership
artifactFamilies map[string]ScriptoriumArtifactFamilyConfig
artifactFamiliesExpanded bool
publishDeclared bool
}
type pipelineProfileSelection struct {
name string
source string
overlayPath string
}
type pipelineFieldOwnership struct {
path string
sources []string
}
type pipelineCompositionEnvelope struct {
imports []string
defaultProfile *string
profiles []pipelineProfileDeclaration
}
type pipelineProfileDeclaration struct {
name string
overlay string
}
// pipelineCompositionSources retains one validated root source set. Each
// selected profile is resolved from a cloned base document so callers can
// safely compare or otherwise resolve multiple profiles without rereading or
// mutating the source set.
type pipelineCompositionSources struct {
rootPath string
envelope pipelineCompositionEnvelope
imports []loadedPipelineImport
overlays []loadedPipelineProfileOverlay
overlaysLoaded bool
base *compositionDocument
}
func loadPipelineCompositionSources(path string) (*pipelineCompositionSources, error) {
rootPath, err := filepath.Abs(path)
if err != nil {
return nil, fmt.Errorf("resolve root pipeline path %q: %w", path, err)
}
rootFile, err := os.Open(rootPath)
if err != nil {
return nil, fmt.Errorf("pipeline file %q: open: %w", path, err)
}
rootDocument, parseErr := parseCompositionDocument(rootPath, rootFile)
closeErr := rootFile.Close()
if parseErr != nil {
return nil, parseErr
}
if closeErr != nil {
return nil, fmt.Errorf("pipeline file %q: close: %w", rootPath, closeErr)
}
baseRoot, envelope, err := splitPipelineCompositionEnvelope(rootDocument)
if err != nil {
return nil, err
}
imports, err := loadPipelineImports(rootPath, envelope.imports)
if err != nil {
return nil, err
}
documents := make([]*compositionDocument, 0, len(imports)+1)
documents = append(documents, baseRoot)
for _, imported := range imports {
documents = append(documents, imported.document)
}
merged, err := mergeAdditiveCompositions(documents...)
if err != nil {
return nil, err
}
return &pipelineCompositionSources{
rootPath: rootPath,
envelope: envelope,
imports: imports,
base: merged,
}, nil
}
func (sources *pipelineCompositionSources) loadOverlays() error {
if sources == nil {
return fmt.Errorf("pipeline composition sources are required")
}
if sources.overlaysLoaded {
return nil
}
overlays, err := loadPipelineProfileOverlays(sources.rootPath, sources.envelope.profiles, sources.imports)
if err != nil {
return err
}
sources.overlays = overlays
sources.overlaysLoaded = true
return nil
}
func (sources *pipelineCompositionSources) resolve(opts PipelineLoadOptions) (*PipelineConfig, error) {
if sources == nil || sources.base == nil {
return nil, fmt.Errorf("pipeline composition sources are required")
}
if !sources.overlaysLoaded {
return nil, fmt.Errorf("pipeline profile overlays have not been loaded")
}
selection, err := selectPipelineProfile(sources.envelope, opts)
if err != nil {
return nil, err
}
merged := &compositionDocument{
root: cloneCompositionNode(sources.base.root),
sources: append([]string(nil), sources.base.sources...),
}
if selection != nil {
overlay, ok := loadedProfileOverlay(sources.overlays, selection.name)
if !ok {
return nil, fmt.Errorf("selected profile %q overlay was not loaded", selection.name)
}
merged, err = mergeOverlayComposition(merged, overlay.document)
if err != nil {
return nil, err
}
selection.overlayPath = overlay.path
}
rendered, err := merged.canonicalYAML()
if err != nil {
return nil, err
}
var cfg PipelineConfig
if err := decodeStrictYAMLFromReader("pipeline", sources.rootPath, strings.NewReader(string(rendered)), &cfg); err != nil {
return nil, fmt.Errorf("assembled pipeline sources %s: %w", formatCompositionSources(merged.sources), err)
}
records, err := merged.semanticRecords()
if err != nil {
return nil, err
}
metadata := &pipelineResolutionMetadata{
rootPath: sources.rootPath,
sources: append([]string(nil), merged.sources...),
selectedProfile: selection,
}
for _, imported := range sources.imports {
metadata.imports = append(metadata.imports, imported.path)
}
for _, record := range records {
metadata.ownership = append(metadata.ownership, pipelineFieldOwnership{
path: record.Path, sources: append([]string(nil), record.Sources...),
})
}
cfg.resolution = metadata
return &cfg, nil
}
func splitPipelineCompositionEnvelope(document *compositionDocument) (*compositionDocument, pipelineCompositionEnvelope, error) {
if err := validateCompositionDocument(document, "root pipeline"); err != nil {
return nil, pipelineCompositionEnvelope{}, err
}
envelope, fieldIndex, err := decodePipelineCompositionEnvelope(document.root)
if err != nil {
return nil, pipelineCompositionEnvelope{}, err
}
if err := validatePipelineCompositionEnvelope(envelope); err != nil {
return nil, pipelineCompositionEnvelope{}, err
}
return pipelineBodyWithoutComposition(document, fieldIndex), envelope, nil
}
func decodePipelineCompositionEnvelope(root *compositionNode) (pipelineCompositionEnvelope, int, error) {
index := compositionFieldIndex(root.fields, "composition")
if index < 0 {
return pipelineCompositionEnvelope{}, -1, nil
}
envelopeNode := root.fields[index].value
if envelopeNode.kind != yaml.MappingNode {
return pipelineCompositionEnvelope{}, -1, fmt.Errorf(
"configuration source %s at composition: expected a mapping, got %s",
formatCompositionSources(envelopeNode.sources), yamlKindName(envelopeNode.kind),
)
}
var envelope pipelineCompositionEnvelope
for _, field := range envelopeNode.fields {
switch field.key {
case "imports":
if field.value.kind != yaml.SequenceNode {
return pipelineCompositionEnvelope{}, -1, fmt.Errorf(
"configuration source %s at composition.imports: expected a list, got %s",
formatCompositionSources(field.value.sources), yamlKindName(field.value.kind),
)
}
for itemIndex, item := range field.value.items {
if item.kind != yaml.ScalarNode || item.tag != "!!str" {
return pipelineCompositionEnvelope{}, -1, fmt.Errorf(
"configuration source %s at composition.imports[%d]: expected a string path",
formatCompositionSources(item.sources), itemIndex,
)
}
envelope.imports = append(envelope.imports, item.value)
}
case "default_profile":
if field.value.kind != yaml.ScalarNode || field.value.tag != "!!str" {
return pipelineCompositionEnvelope{}, -1, fmt.Errorf(
"configuration source %s at composition.default_profile: expected a string profile name",
formatCompositionSources(field.value.sources),
)
}
name, err := normalizePipelineProfileName(field.value.value, "composition.default_profile")
if err != nil {
return pipelineCompositionEnvelope{}, -1, err
}
envelope.defaultProfile = &name
case "profiles":
if field.value.kind != yaml.MappingNode {
return pipelineCompositionEnvelope{}, -1, fmt.Errorf(
"configuration source %s at composition.profiles: expected a mapping, got %s",
formatCompositionSources(field.value.sources), yamlKindName(field.value.kind),
)
}
if len(field.value.fields) == 0 {
return pipelineCompositionEnvelope{}, -1, fmt.Errorf("composition.profiles must declare at least one named profile")
}
for _, profileField := range field.value.fields {
name, err := normalizePipelineProfileName(profileField.key, "composition.profiles profile name")
if err != nil {
return pipelineCompositionEnvelope{}, -1, err
}
profile, err := parsePipelineProfileDeclaration(name, profileField.value)
if err != nil {
return pipelineCompositionEnvelope{}, -1, err
}
envelope.profiles = append(envelope.profiles, profile)
}
default:
return pipelineCompositionEnvelope{}, -1, fmt.Errorf(
"configuration source %s at composition.%s: unknown composition field %q",
formatCompositionSources(field.value.sources), field.key, field.key,
)
}
}
return envelope, index, nil
}
func validatePipelineCompositionEnvelope(envelope pipelineCompositionEnvelope) error {
if envelope.defaultProfile != nil && !pipelineProfileDeclared(envelope.profiles, *envelope.defaultProfile) {
return fmt.Errorf(
"composition.default_profile %q does not name a declared profile", *envelope.defaultProfile,
)
}
return nil
}
func pipelineBodyWithoutComposition(document *compositionDocument, compositionIndex int) *compositionDocument {
root := cloneCompositionNode(document.root)
if compositionIndex >= 0 {
root.fields = append(root.fields[:compositionIndex], root.fields[compositionIndex+1:]...)
}
return &compositionDocument{root: root, sources: append([]string(nil), document.sources...)}
}
func parsePipelineProfileDeclaration(name string, node *compositionNode) (pipelineProfileDeclaration, error) {
path := "composition.profiles." + name
if node.kind != yaml.MappingNode {
return pipelineProfileDeclaration{}, fmt.Errorf(
"configuration source %s at %s: expected a mapping, got %s",
formatCompositionSources(node.sources), path, yamlKindName(node.kind),
)
}
profile := pipelineProfileDeclaration{name: name}
for _, field := range node.fields {
if field.key != "overlay" {
return pipelineProfileDeclaration{}, fmt.Errorf(
"configuration source %s at %s.%s: unknown profile field %q; only overlay is supported",
formatCompositionSources(field.value.sources), path, field.key, field.key,
)
}
if field.value.kind != yaml.ScalarNode || field.value.tag != "!!str" {
return pipelineProfileDeclaration{}, fmt.Errorf(
"configuration source %s at %s.overlay: expected a string path",
formatCompositionSources(field.value.sources), path,
)
}
profile.overlay = field.value.value
}
if profile.overlay == "" {
return pipelineProfileDeclaration{}, fmt.Errorf("%s.overlay is required", path)
}
return profile, nil
}
func normalizePipelineProfileName(value, label string) (string, error) {
if value == "" || strings.TrimSpace(value) != value {
return "", fmt.Errorf("%s must be non-empty without surrounding whitespace", label)
}
for _, character := range value {
if unicode.IsControl(character) {
return "", fmt.Errorf("%s %q must not contain control characters", label, value)
}
}
return value, nil
}
func pipelineProfileDeclared(profiles []pipelineProfileDeclaration, name string) bool {
for _, profile := range profiles {
if profile.name == name {
return true
}
}
return false
}
func selectPipelineProfile(envelope pipelineCompositionEnvelope, opts PipelineLoadOptions) (*pipelineProfileSelection, error) {
if opts.Profile != nil {
name, err := normalizePipelineProfileName(*opts.Profile, "explicit profile selection")
if err != nil {
return nil, err
}
if len(envelope.profiles) == 0 {
return nil, fmt.Errorf("explicit profile %q was selected but the pipeline declares no profiles", name)
}
if !pipelineProfileDeclared(envelope.profiles, name) {
return nil, fmt.Errorf("explicit profile %q is not declared by the pipeline", name)
}
return &pipelineProfileSelection{name: name, source: "cli"}, nil
}
if len(envelope.profiles) == 0 {
return nil, nil
}
if envelope.defaultProfile == nil {
return nil, fmt.Errorf("pipeline declares profiles but composition.default_profile is omitted and no profile was explicitly selected")
}
return &pipelineProfileSelection{name: *envelope.defaultProfile, source: "default"}, nil
}
type loadedPipelineImport struct {
path string
document *compositionDocument
info os.FileInfo
}
type loadedPipelineProfileOverlay struct {
profile string
path string
document *compositionDocument
info os.FileInfo
}
func loadPipelineProfileOverlays(
rootPath string,
declared []pipelineProfileDeclaration,
imports []loadedPipelineImport,
) ([]loadedPipelineProfileOverlay, error) {
if len(declared) == 0 {
return nil, nil
}
rootDir := filepath.Dir(rootPath)
rootInfo, err := os.Stat(rootPath)
if err != nil {
return nil, fmt.Errorf("inspect root pipeline file %q: %w", rootPath, err)
}
seenPaths := make(map[string]string, len(declared))
loaded := make([]loadedPipelineProfileOverlay, 0, len(declared))
for _, profile := range declared {
label := "composition.profiles." + profile.name + ".overlay"
raw := profile.overlay
if strings.TrimSpace(raw) != raw || raw == "" {
return nil, fmt.Errorf("%s must be a non-empty path without surrounding whitespace", label)
}
normalized, err := pathsafe.NormalizeRelativeDestination(raw)
if err != nil {
return nil, fmt.Errorf("%s path %q is invalid: %w", label, raw, err)
}
extension := filepath.Ext(filepath.FromSlash(normalized))
if extension != ".yml" && extension != ".yaml" {
return nil, fmt.Errorf("%s path %q must use .yml or .yaml", label, raw)
}
if prior, duplicate := seenPaths[normalized]; duplicate {
return nil, fmt.Errorf("%s path %q duplicates profile %q overlay after normalization", label, raw, prior)
}
seenPaths[normalized] = profile.name
resolved := filepath.Join(rootDir, filepath.FromSlash(normalized))
file, err := os.Open(resolved)
if err != nil {
return nil, fmt.Errorf("open %s path %q beneath root pipeline directory: %w", label, raw, err)
}
info, statErr := file.Stat()
if statErr != nil {
_ = file.Close()
return nil, fmt.Errorf("inspect %s path %q: %w", label, raw, statErr)
}
if os.SameFile(rootInfo, info) {
_ = file.Close()
return nil, fmt.Errorf("%s path %q references the root pipeline itself", label, raw)
}
for _, imported := range imports {
if os.SameFile(imported.info, info) {
_ = file.Close()
return nil, fmt.Errorf("%s path %q references the same file as imported source %q", label, raw, imported.path)
}
}
for _, prior := range loaded {
if os.SameFile(prior.info, info) {
_ = file.Close()
return nil, fmt.Errorf("%s path %q references the same file as profile %q overlay", label, raw, prior.profile)
}
}
document, parseErr := parseCompositionDocument(resolved, file)
closeErr := file.Close()
if parseErr != nil {
return nil, parseErr
}
if closeErr != nil {
return nil, fmt.Errorf("close %s path %q: %w", label, raw, closeErr)
}
if compositionFieldIndex(document.root.fields, "composition") >= 0 {
return nil, fmt.Errorf("profile overlay source %q declares composition; only the root pipeline may declare composition", resolved)
}
loaded = append(loaded, loadedPipelineProfileOverlay{
profile: profile.name, path: resolved, document: document, info: info,
})
}
return loaded, nil
}
func loadedProfileOverlay(overlays []loadedPipelineProfileOverlay, name string) (loadedPipelineProfileOverlay, bool) {
for _, overlay := range overlays {
if overlay.profile == name {
return overlay, true
}
}
return loadedPipelineProfileOverlay{}, false
}
func loadPipelineImports(rootPath string, declared []string) ([]loadedPipelineImport, error) {
if len(declared) == 0 {
return nil, nil
}
rootDir := filepath.Dir(rootPath)
rootInfo, err := os.Stat(rootPath)
if err != nil {
return nil, fmt.Errorf("inspect root pipeline file %q: %w", rootPath, err)
}
seenPaths := make(map[string]int, len(declared))
loaded := make([]loadedPipelineImport, 0, len(declared))
for index, raw := range declared {
if strings.TrimSpace(raw) != raw || raw == "" {
return nil, fmt.Errorf("composition.imports[%d] must be a non-empty path without surrounding whitespace", index)
}
normalized, err := pathsafe.NormalizeRelativeDestination(raw)
if err != nil {
return nil, fmt.Errorf("composition.imports[%d] path %q is invalid: %w", index, raw, err)
}
extension := filepath.Ext(filepath.FromSlash(normalized))
if extension != ".yml" && extension != ".yaml" {
return nil, fmt.Errorf("composition.imports[%d] path %q must use .yml or .yaml", index, raw)
}
if prior, duplicate := seenPaths[normalized]; duplicate {
return nil, fmt.Errorf(
"composition.imports[%d] path %q duplicates composition.imports[%d] after normalization",
index, raw, prior,
)
}
seenPaths[normalized] = index
resolved := filepath.Join(rootDir, filepath.FromSlash(normalized))
if resolved == rootPath {
return nil, fmt.Errorf("composition.imports[%d] path %q imports the root pipeline itself", index, raw)
}
file, err := os.Open(resolved)
if err != nil {
return nil, fmt.Errorf("open composition.imports[%d] path %q beneath root pipeline directory: %w", index, raw, err)
}
info, statErr := file.Stat()
if statErr != nil {
_ = file.Close()
return nil, fmt.Errorf("inspect composition.imports[%d] path %q: %w", index, raw, statErr)
}
if os.SameFile(rootInfo, info) {
_ = file.Close()
return nil, fmt.Errorf("composition.imports[%d] path %q imports the root pipeline itself", index, raw)
}
for priorIndex, prior := range loaded {
if os.SameFile(prior.info, info) {
_ = file.Close()
return nil, fmt.Errorf(
"composition.imports[%d] path %q references the same file as composition.imports[%d] %q",
index, raw, priorIndex, declared[priorIndex],
)
}
}
document, parseErr := parseCompositionDocument(resolved, file)
closeErr := file.Close()
if parseErr != nil {
return nil, parseErr
}
if closeErr != nil {
return nil, fmt.Errorf("close composition.imports[%d] path %q: %w", index, raw, closeErr)
}
if compositionFieldIndex(document.root.fields, "composition") >= 0 {
return nil, fmt.Errorf(
"imported configuration source %q declares composition; only the root pipeline may declare composition",
resolved,
)
}
loaded = append(loaded, loadedPipelineImport{path: resolved, document: document, info: info})
}
return loaded, nil
}

View File

@@ -1,431 +0,0 @@
package config
import (
"os"
"path/filepath"
"reflect"
"slices"
"strings"
"testing"
)
func TestSplitPipelineCompositionEnvelopePreservesInputAndDeclarationOrder(t *testing.T) {
document, err := parseCompositionBytes("pipeline.yml", []byte(`composition:
imports: [first.yml, second.yml]
default_profile: production
profiles:
production:
overlay: production.yml
testing:
overlay: testing.yml
workspace:
root: /srv/narratio
`))
if err != nil {
t.Fatal(err)
}
originalRoot := cloneCompositionNode(document.root)
originalSources := append([]string(nil), document.sources...)
body, envelope, err := splitPipelineCompositionEnvelope(document)
if err != nil {
t.Fatalf("splitPipelineCompositionEnvelope() error = %v", err)
}
if !reflect.DeepEqual(document.root, originalRoot) || !slices.Equal(document.sources, originalSources) {
t.Fatalf("splitPipelineCompositionEnvelope() mutated its input")
}
if compositionFieldIndex(body.root.fields, "composition") >= 0 {
t.Fatalf("pipeline body still contains composition envelope")
}
if compositionFieldIndex(body.root.fields, "workspace") < 0 {
t.Fatalf("pipeline body lost ordinary root fields")
}
if want := []string{"first.yml", "second.yml"}; !slices.Equal(envelope.imports, want) {
t.Fatalf("imports = %#v, want %#v", envelope.imports, want)
}
if len(envelope.profiles) != 2 || envelope.profiles[0].name != "production" || envelope.profiles[1].name != "testing" {
t.Fatalf("profiles = %#v, want declaration order", envelope.profiles)
}
}
func TestSplitPipelineCompositionEnvelopeRejectsMalformedShapesWithoutMutation(t *testing.T) {
tests := []struct {
name string
yaml string
want string
}{
{name: "envelope", yaml: "composition: []\n", want: "expected a mapping"},
{name: "imports mapping", yaml: "composition:\n imports: {}\n", want: "expected a list"},
{name: "import value", yaml: "composition:\n imports: [1]\n", want: "expected a string path"},
{name: "default profile", yaml: "composition:\n default_profile: {}\n", want: "expected a string profile name"},
{name: "profiles sequence", yaml: "composition:\n profiles: []\n", want: "expected a mapping"},
{name: "empty profiles", yaml: "composition:\n profiles: {}\n", want: "at least one named profile"},
{name: "profile declaration", yaml: "composition:\n profiles:\n production: production.yml\n", want: "expected a mapping"},
{name: "invalid profile name", yaml: "composition:\n profiles:\n ' production ':\n overlay: production.yml\n", want: "surrounding whitespace"},
{name: "unknown field", yaml: "composition:\n unexpected: true\n", want: "unknown composition field"},
{name: "unknown default", yaml: "composition:\n default_profile: testing\n profiles:\n production:\n overlay: production.yml\n", want: "does not name a declared profile"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
document, err := parseCompositionBytes("pipeline.yml", []byte(test.yaml))
if err != nil {
t.Fatal(err)
}
originalRoot := cloneCompositionNode(document.root)
_, _, err = splitPipelineCompositionEnvelope(document)
if err == nil || !strings.Contains(err.Error(), test.want) {
t.Fatalf("splitPipelineCompositionEnvelope() error = %v, want containing %q", err, test.want)
}
if !reflect.DeepEqual(document.root, originalRoot) {
t.Fatalf("splitPipelineCompositionEnvelope() mutated invalid input")
}
})
}
}
func TestLoadPipelineCompositionImportsDisjointFieldsAndTracksOwnership(t *testing.T) {
dir := t.TempDir()
rootPath := writePipelineSource(t, dir, "pipeline.yml", `composition:
imports:
- conf.d/platform.yml
- conf.d/artifacts.yml
whisperx:
transcribe_url: https://transcription.example.com/transcribe
`)
platformPath := writePipelineSource(t, dir, "conf.d/platform.yml", `workspace:
root: /srv/narratio/work
storage:
backend: local
`)
artifactsPath := writePipelineSource(t, dir, "conf.d/artifacts.yml", `scriptorium:
artifacts:
player_handout:
enabled: false
session_recap:
enabled: true
prompt_id: dnd.session_recap
output_path: artifacts/session_recap.md
`)
cfg, err := LoadPipeline(rootPath)
if err != nil {
t.Fatalf("LoadPipeline() error = %v", err)
}
if cfg.Workspace.Root != "/srv/narratio/work" || cfg.Storage.Backend != StorageBackendLocal {
t.Fatalf("imported platform config = workspace=%q storage=%#v", cfg.Workspace.Root, cfg.Storage)
}
if cfg.Scriptorium == nil || len(cfg.Scriptorium.Artifacts) != 2 || cfg.Scriptorium.Artifacts["player_handout"].Enabled {
t.Fatalf("imported artifacts = %#v", cfg.Scriptorium)
}
if cfg.WhisperX.TranscribeURL != "https://transcription.example.com/transcribe" {
t.Fatalf("root field = %q", cfg.WhisperX.TranscribeURL)
}
if cfg.resolution == nil {
t.Fatal("pipeline resolution metadata = nil")
}
wantSources := []string{absolutePath(t, rootPath), absolutePath(t, platformPath), absolutePath(t, artifactsPath)}
if !slices.Equal(cfg.resolution.sources, wantSources) || !slices.Equal(cfg.resolution.imports, wantSources[1:]) {
t.Fatalf("resolution sources=%#v imports=%#v, want %#v / %#v", cfg.resolution.sources, cfg.resolution.imports, wantSources, wantSources[1:])
}
assertPipelineFieldOwner(t, cfg, "whisperx.transcribe_url", absolutePath(t, rootPath))
assertPipelineFieldOwner(t, cfg, "workspace.root", absolutePath(t, platformPath))
assertPipelineFieldOwner(t, cfg, "scriptorium.artifacts.session_recap.prompt_id", absolutePath(t, artifactsPath))
}
func TestLoadPipelineCompositionMergesDisjointKeyedEntries(t *testing.T) {
dir := t.TempDir()
rootPath := writePipelineSource(t, dir, "pipeline.yml", `composition:
imports: [first.yml, second.yml]
whisperx:
transcribe_url: https://transcription.example.com/transcribe
scriptorium:
artifacts:
root_artifact:
enabled: false
`)
writePipelineSource(t, dir, "first.yml", `scriptorium:
artifacts:
first_artifact:
enabled: false
`)
writePipelineSource(t, dir, "second.yml", `scriptorium:
artifacts:
second_artifact:
enabled: false
`)
cfg, err := LoadPipeline(rootPath)
if err != nil {
t.Fatal(err)
}
if got := len(cfg.Scriptorium.Artifacts); got != 3 {
t.Fatalf("artifact count = %d, want 3: %#v", got, cfg.Scriptorium.Artifacts)
}
}
func TestLoadPipelineCompositionRejectsBaseConflictsWithAllSources(t *testing.T) {
tests := []struct {
name string
root string
imports map[string]string
path string
sources []string
}{
{
name: "root and import identical scalar",
root: "whisperx:\n language: en\n",
imports: map[string]string{"one.yml": "whisperx:\n language: en\n"},
path: "whisperx.language", sources: []string{"pipeline.yml", "one.yml"},
},
{
name: "all import claimants",
imports: map[string]string{
"one.yml": "workspace:\n root: /one\n",
"two.yml": "workspace:\n root: /two\n",
"three.yml": "workspace:\n root: /three\n",
},
path: "workspace.root", sources: []string{"one.yml", "two.yml", "three.yml"},
},
{
name: "atomic list",
root: "audita:\n modules: [one]\n",
imports: map[string]string{"one.yml": "audita:\n modules: [two]\n"},
path: "audita.modules", sources: []string{"pipeline.yml", "one.yml"},
},
{
name: "kind conflict",
root: "workspace:\n root: /work\n",
imports: map[string]string{"one.yml": "workspace: invalid\n"},
path: "workspace", sources: []string{"pipeline.yml", "one.yml"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
dir := t.TempDir()
order := make([]string, 0, len(tt.imports))
for _, name := range []string{"one.yml", "two.yml", "three.yml"} {
if _, ok := tt.imports[name]; ok {
order = append(order, name)
}
}
root := "composition:\n imports:\n"
for _, name := range order {
root += " - " + name + "\n"
}
root += tt.root
rootPath := writePipelineSource(t, dir, "pipeline.yml", root)
for name, content := range tt.imports {
writePipelineSource(t, dir, name, content)
}
_, err := LoadPipeline(rootPath)
if err == nil {
t.Fatal("LoadPipeline() error = nil")
}
if !strings.Contains(err.Error(), tt.path) {
t.Fatalf("error = %q, want path %q", err, tt.path)
}
for _, source := range tt.sources {
if !strings.Contains(err.Error(), source) {
t.Fatalf("error = %q, want source %q", err, source)
}
}
})
}
}
func TestLoadPipelineCompositionRejectsUnsafeOrInvalidImports(t *testing.T) {
tests := []struct {
name string
imports []string
setup func(*testing.T, string)
want string
}{
{name: "empty", imports: []string{""}, want: "non-empty path"},
{name: "surrounding whitespace", imports: []string{" one.yml "}, want: "surrounding whitespace"},
{name: "absolute", imports: []string{"/tmp/one.yml"}, want: "invalid"},
{name: "traversal", imports: []string{"../one.yml"}, want: "invalid"},
{name: "unsupported extension", imports: []string{"one.json"}, want: ".yml or .yaml"},
{name: "missing", imports: []string{"missing.yml"}, want: "open composition.imports"},
{name: "duplicate normalized", imports: []string{"one.yml", "./one.yml"}, setup: func(t *testing.T, dir string) {
writePipelineSource(t, dir, "one.yml", "workspace:\n root: /work\n")
}, want: "duplicates composition.imports"},
{name: "root self import", imports: []string{"pipeline.yml"}, want: "root pipeline itself"},
{name: "directory", imports: []string{"directory.yml"}, setup: func(t *testing.T, dir string) {
if err := os.Mkdir(filepath.Join(dir, "directory.yml"), 0o755); err != nil {
t.Fatal(err)
}
}, want: "is a directory"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
dir := t.TempDir()
if tt.setup != nil {
tt.setup(t, dir)
}
rootPath := writeImportRoot(t, dir, tt.imports)
_, err := LoadPipeline(rootPath)
if err == nil || !strings.Contains(strings.ToLower(err.Error()), strings.ToLower(tt.want)) {
t.Fatalf("LoadPipeline() error = %v, want containing %q", err, tt.want)
}
})
}
}
func TestLoadPipelineCompositionRejectsSameFileAliasesAndImportedComposition(t *testing.T) {
t.Run("same file through hard link", func(t *testing.T) {
dir := t.TempDir()
writePipelineSource(t, dir, "one.yml", "workspace:\n root: /work\n")
if err := os.Link(filepath.Join(dir, "one.yml"), filepath.Join(dir, "two.yml")); err != nil {
t.Skipf("hard links unavailable: %v", err)
}
rootPath := writeImportRoot(t, dir, []string{"one.yml", "two.yml"})
_, err := LoadPipeline(rootPath)
if err == nil || !strings.Contains(err.Error(), "same file") {
t.Fatalf("LoadPipeline() error = %v, want same-file rejection", err)
}
})
t.Run("imported composition", func(t *testing.T) {
dir := t.TempDir()
rootPath := writeImportRoot(t, dir, []string{"nested.yml"})
writePipelineSource(t, dir, "nested.yml", "composition:\n imports: []\n")
_, err := LoadPipeline(rootPath)
if err == nil || !strings.Contains(err.Error(), "only the root pipeline") || !strings.Contains(err.Error(), "nested.yml") {
t.Fatalf("LoadPipeline() error = %v, want imported composition rejection", err)
}
})
t.Run("empty profiles", func(t *testing.T) {
dir := t.TempDir()
rootPath := writePipelineSource(t, dir, "pipeline.yml", "composition:\n profiles: {}\n")
_, err := LoadPipeline(rootPath)
if err == nil || !strings.Contains(err.Error(), "must declare at least one named profile") {
t.Fatalf("LoadPipeline() error = %v, want empty profiles rejection", err)
}
})
}
func TestLoadPipelineCompositionReportsImportedParseAndSchemaSources(t *testing.T) {
t.Run("malformed imported YAML", func(t *testing.T) {
dir := t.TempDir()
rootPath := writeImportRoot(t, dir, []string{"broken.yml"})
brokenPath := writePipelineSource(t, dir, "broken.yml", "workspace: [\n")
_, err := LoadPipeline(rootPath)
if err == nil || !strings.Contains(err.Error(), absolutePath(t, brokenPath)) || !strings.Contains(err.Error(), "decode YAML") {
t.Fatalf("LoadPipeline() error = %v, want imported parse source", err)
}
})
t.Run("unknown imported field", func(t *testing.T) {
dir := t.TempDir()
rootPath := writeImportRoot(t, dir, []string{"unknown.yml"})
unknownPath := writePipelineSource(t, dir, "unknown.yml", "unknown_field: true\n")
_, err := LoadPipeline(rootPath)
if err == nil || !strings.Contains(err.Error(), absolutePath(t, unknownPath)) || !strings.Contains(err.Error(), "strict decode failed") {
t.Fatalf("LoadPipeline() error = %v, want assembled source-aware strict error", err)
}
})
}
func TestLoadPipelineCompositionKeepsRelativePathsRootBased(t *testing.T) {
rootDir := t.TempDir()
monolithicPath := writePipelineSource(t, rootDir, "monolithic.yml", testPipelineBaseYAML+`
notarius:
enabled: true
config_path: tool/notarius.yml
pipeline_id: dnd-session
outputs:
npc_registry:
lane_id: npc-registry
media_type: application/json
schema_id: notarius.dnd.npc_registry
schema_version: v1
`)
composedPath := writePipelineSource(t, rootDir, "pipeline.yml", `composition:
imports: [conf.d/extraction.yml]
`+testPipelineBaseYAML)
writePipelineSource(t, rootDir, "conf.d/extraction.yml", `notarius:
enabled: true
config_path: tool/notarius.yml
pipeline_id: dnd-session
outputs:
npc_registry:
lane_id: npc-registry
media_type: application/json
schema_id: notarius.dnd.npc_registry
schema_version: v1
`)
monolithic, err := LoadPipeline(monolithicPath)
if err != nil {
t.Fatal(err)
}
composed, err := LoadPipeline(composedPath)
if err != nil {
t.Fatal(err)
}
want := filepath.Join(rootDir, "tool", "notarius.yml")
if monolithic.Notarius.ConfigPath != want || composed.Notarius.ConfigPath != want {
t.Fatalf("config paths = monolithic %q composed %q, want %q", monolithic.Notarius.ConfigPath, composed.Notarius.ConfigPath, want)
}
if monolithic.Notarius.WorkingDirectory != filepath.Dir(want) || composed.Notarius.WorkingDirectory != filepath.Dir(want) {
t.Fatalf("working directories = %q / %q", monolithic.Notarius.WorkingDirectory, composed.Notarius.WorkingDirectory)
}
}
func writeImportRoot(t *testing.T, dir string, imports []string) string {
t.Helper()
var builder strings.Builder
builder.WriteString("composition:\n imports:\n")
for _, imported := range imports {
builder.WriteString(" - ")
if imported == "" {
builder.WriteString(`""`)
} else {
builder.WriteString(`"` + imported + `"`)
}
builder.WriteByte('\n')
}
builder.WriteString("whisperx:\n transcribe_url: https://transcription.example.com/transcribe\n")
return writePipelineSource(t, dir, "pipeline.yml", builder.String())
}
func writePipelineSource(t *testing.T, root, relative, content string) string {
t.Helper()
path := filepath.Join(root, filepath.FromSlash(relative))
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
t.Fatal(err)
}
return path
}
func absolutePath(t *testing.T, path string) string {
t.Helper()
absolute, err := filepath.Abs(path)
if err != nil {
t.Fatal(err)
}
return absolute
}
func assertPipelineFieldOwner(t *testing.T, cfg *PipelineConfig, path, source string) {
t.Helper()
if cfg == nil || cfg.resolution == nil {
t.Fatal("pipeline resolution metadata is absent")
}
for _, ownership := range cfg.resolution.ownership {
if ownership.path == path {
if !slices.Equal(ownership.sources, []string{source}) {
t.Fatalf("owner of %s = %#v, want %q", path, ownership.sources, source)
}
return
}
}
t.Fatalf("ownership path %q not found: %#v", path, cfg.resolution.ownership)
}

View File

@@ -0,0 +1,391 @@
package config
import (
"bytes"
"errors"
"fmt"
"path/filepath"
"reflect"
"sort"
"strings"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
)
type pipelineResolutionMetadata struct {
rootPath string
imports []string
sources []string
selectedProfile *pipelineProfileSelection
effectiveDigest string
logicalNotariusConfig string
logicalNotariusWorking string
logicalNotariusCaptured bool
ownership []pipelineFieldOwnership
artifactFamilies map[string]ScriptoriumArtifactFamilyConfig
artifactFamiliesExpanded bool
publishDeclared bool
configkitMetadata configkit.Metadata
configkitSources []configkit.SourceInfo
}
type pipelineProfileSelection struct {
name string
source string
overlayPath string
}
type pipelineFieldOwnership struct {
path string
sources []string
}
func loadPipelineWithConfigKit(path string, opts PipelineLoadOptions) (*PipelineConfig, error) {
profile := selectedPipelineProfileName(opts)
result, err := configkit.Load(configkit.File(path), configkit.Options[PipelineConfig]{
Imports: true,
Profile: profile,
Definition: configkit.Definition[PipelineConfig]{
Defaults: pipelineConfigDefaults,
Resolve: resolvePipelineConfig,
Validate: validatePipelineConfigKit,
},
})
if err != nil {
return nil, describePipelineLoadError(err)
}
cfg := result.Config
metadata, err := pipelineMetadataFromConfigKit(&cfg, result, opts)
if err != nil {
return nil, fmt.Errorf("load pipeline config: %w", err)
}
cfg.resolution = metadata
return &cfg, nil
}
func describePipelineLoadError(err error) error {
var loadError *configkit.LoadError
if !errors.As(err, &loadError) {
return fmt.Errorf("load pipeline config: %w", err)
}
var diagnostics bytes.Buffer
resolved := loadError.Diagnostics()
sourcePaths := make(map[string]string, len(loadError.Sources()))
for _, source := range loadError.Sources() {
if source.Path != "" {
sourcePaths[source.ID] = source.Path
} else if source.Label != "" {
sourcePaths[source.ID] = source.Label
}
}
for index := range resolved {
if resolved[index].Primary != nil {
if name := sourcePaths[resolved[index].Primary.SourceID]; name != "" {
resolved[index].Primary.SourceID = name
}
}
for relatedIndex := range resolved[index].Related {
if name := sourcePaths[resolved[index].Related[relatedIndex].SourceID]; name != "" {
resolved[index].Related[relatedIndex].SourceID = name
}
}
}
if writeErr := configkit.WriteDiagnostics(&diagnostics, resolved); writeErr != nil {
return fmt.Errorf("load pipeline config: %w", err)
}
return fmt.Errorf("load pipeline config: strict decode failed or validation failed: %w: %s", err, strings.TrimSpace(diagnostics.String()))
}
func selectedPipelineProfileName(opts PipelineLoadOptions) string {
if opts.Profile == nil || *opts.Profile == "" || *opts.Profile == "default" {
return ""
}
return *opts.Profile
}
func pipelineConfigDefaults() PipelineConfig {
return PipelineConfig{
Workspace: WorkspaceConfig{Root: DefaultWorkspaceRoot},
Campaigns: CampaignsConfig{Root: DefaultCampaignsRoot},
Storage: StorageConfig{Backend: StorageBackendLocal},
Spool: SpoolConfig{Root: DefaultSpoolRoot},
Cache: CacheConfig{Root: DefaultCacheRoot, S3Audio: boolPtr(DefaultCacheS3Audio)},
Publish: &PublishConfig{
Enabled: boolPtr(DefaultArchiveEnabled), UploadRun: boolPtr(DefaultArchiveUploadRun),
Outputs: defaultPublishOutputs(),
},
WhisperX: WhisperXConfig{
Language: DefaultWhisperXLanguage, Timeout: DefaultWhisperXTimeout,
RetryDelay: DefaultWhisperXRetryDelay, Concurrency: intPtr(DefaultWhisperXConcurrency),
Retries: intPtr(DefaultWhisperXRetries),
},
Seriatim: SeriatimConfig{
Binary: DefaultSeriatimBinary, Timeout: DefaultSeriatimTimeout,
OutputSchema: DefaultSeriatimOutputSchema, CoalesceGap: float64Ptr(DefaultSeriatimCoalesceGap),
Report: boolPtr(DefaultSeriatimReport),
},
Audita: AuditaConfig{Binary: DefaultAuditaBinary, Timeout: DefaultAuditaTimeout, Report: boolPtr(DefaultAuditaReport)},
Normalize: &NormalizeConfig{OutputPath: DefaultNormalizeOutputPath, OutputSchema: DefaultNormalizeOutputSchema, Report: boolPtr(DefaultNormalizeReport)},
Trim: &TrimConfig{
Enabled: boolPtr(DefaultTrimEnabled), OutputPath: DefaultTrimOutputPath,
Bounds: TrimBoundsConfig{
PromptID: DefaultTrimBoundsPromptID, TranscriptInputName: DefaultTrimBoundsTranscriptInputName,
OutputPath: DefaultTrimBoundsOutputPath, Timeout: DefaultTrimBoundsTimeout,
},
Seriatim: TrimSeriatimConfig{Report: boolPtr(DefaultTrimSeriatimReport)},
},
Render: &RenderConfig{
Enabled: boolPtr(DefaultRenderEnabled), Format: DefaultRenderFormat, Title: DefaultRenderTitle,
IncludeTimestamps: boolPtr(DefaultRenderTimestamps), IncludeSegmentIDs: boolPtr(DefaultRenderSegmentIDs),
},
Notification: NotificationConfig{Mode: DefaultNotificationMode},
}
}
func defaultPublishOutputs() []PublishOutputRule {
outputs := append([]PublishOutputRule(nil), DefaultPublishOutputs...)
for index := range outputs {
outputs[index].Required = boolPtr(true)
}
return outputs
}
func resolvePipelineConfig(cfg *PipelineConfig, resolution *configkit.Resolution) error {
backend := strings.ToLower(strings.TrimSpace(cfg.Storage.Backend))
if backend != cfg.Storage.Backend {
configkit.Normalize(resolution, configkit.Keys("storage", "backend"), &cfg.Storage.Backend, backend, "canonical storage backend")
}
if cfg.Storage.S3 != nil {
path := configkit.Keys("storage", "s3")
configkit.Default(resolution, path.Key("root_prefix"), &cfg.Storage.S3.RootPrefix, DefaultStorageS3RootPrefix, "default S3 root prefix")
configkit.Default(resolution, path.Key("access_key_id_env"), &cfg.Storage.S3.AccessKeyIDEnv, DefaultS3AccessKeyIDEnv, "default S3 access-key reference")
configkit.Default(resolution, path.Key("secret_access_key_env"), &cfg.Storage.S3.SecretKeyEnv, DefaultS3SecretAccessKeyEnv, "default S3 secret-key reference")
}
if cfg.Trim != nil && cfg.Trim.Enabled != nil && *cfg.Trim.Enabled && cfg.Scriptorium == nil {
value := &ScriptoriumConfig{Binary: DefaultScriptoriumBinary, Timeout: DefaultScriptoriumTimeout}
configkit.Default(resolution, configkit.Keys("scriptorium"), &cfg.Scriptorium, value, "trim requires Scriptorium")
}
if cfg.Scriptorium != nil {
path := configkit.Keys("scriptorium")
configkit.Default(resolution, path.Key("binary"), &cfg.Scriptorium.Binary, DefaultScriptoriumBinary, "default Scriptorium binary")
configkit.Default(resolution, path.Key("timeout"), &cfg.Scriptorium.Timeout, DefaultScriptoriumTimeout, "default Scriptorium timeout")
}
if cfg.Notarius != nil {
path := configkit.Keys("notarius")
configkit.Default(resolution, path.Key("binary"), &cfg.Notarius.Binary, DefaultNotariusBinary, "default Notarius binary")
configkit.Default(resolution, path.Key("timeout"), &cfg.Notarius.Timeout, DefaultNotariusTimeout, "default Notarius timeout")
if cfg.Notarius.Enabled && cfg.Notarius.ConfigPath != "" {
configkit.Default(resolution, path.Key("working_directory"), &cfg.Notarius.WorkingDirectory,
filepath.Dir(cfg.Notarius.ConfigPath), "derived from Notarius config path", path.Key("config_path"))
}
}
if cfg.Publish != nil {
defaultedOutput := false
for index := range cfg.Publish.Outputs {
output := cfg.Publish.Outputs[index]
path := configkit.Keys("publish", "outputs").Index(index).Key("required")
if !resolution.IsExplicit(path) {
defaultedOutput = true
}
configkit.Default(resolution, path, &output.Required, boolPtr(true), "publish outputs are required by default")
cfg.Publish.Outputs[index] = output
}
if defaultedOutput {
resolution.Record(configkit.Keys("publish", "outputs"), configkit.ChangeDerivedDefault, "publish output entry defaults")
}
}
before := cloneNotariusForResolution(cfg.Notarius)
if err := NormalizePipelineConfig(cfg); err != nil {
resolution.Error(configkit.Keys("notarius"), err.Error())
return nil
}
if !reflect.DeepEqual(before, cfg.Notarius) {
resolution.Record(configkit.Keys("notarius"), configkit.ChangeNormalization, "canonical Notarius configuration")
}
if cfg.Publish != nil {
configuredOutputs := scriptoriumOutputPathMap(cfg.Scriptorium)
extractionOutputs := notariusOutputKeySet(cfg.Notarius)
for index := range cfg.Publish.Outputs {
output := cfg.Publish.Outputs[index]
if strings.TrimSpace(output.Dest) != "" {
continue
}
destination, err := artifactpolicy.ResolvePublishedDestinationWithExtractions(strings.TrimSpace(output.Source), "", configuredOutputs, extractionOutputs)
if err != nil {
continue
}
configkit.Normalize(resolution, configkit.Keys("publish", "outputs").Index(index).Key("dest"), &output.Dest, destination, "derived from publish source")
cfg.Publish.Outputs[index] = output
}
}
return nil
}
func cloneNotariusForResolution(value *NotariusConfig) *NotariusConfig {
if value == nil {
return nil
}
clone := *value
clone.References = cloneStringMap(value.References)
if value.Outputs != nil {
clone.Outputs = make(map[string]NotariusOutputConfig, len(value.Outputs))
for key, output := range value.Outputs {
clone.Outputs[key] = output
}
}
return &clone
}
func cloneStringMap(value map[string]string) map[string]string {
if value == nil {
return nil
}
clone := make(map[string]string, len(value))
for key, item := range value {
clone[key] = item
}
return clone
}
func validatePipelineConfigKit(cfg *PipelineConfig, validation *configkit.Validation) {
// Checks that depend on family expansion or campaign/session context remain
// at Narratio's explicit resolution boundary. This early definition check is
// deliberately limited to an invariant that is wholly file-local.
if strings.TrimSpace(cfg.Workspace.Root) == "" {
validation.Error(configkit.Keys("workspace", "root"), "workspace root is required")
}
}
func pipelineMetadataFromConfigKit(cfg *PipelineConfig, result *configkit.Result[PipelineConfig], opts PipelineLoadOptions) (*pipelineResolutionMetadata, error) {
sources := result.Sources()
metadata := result.Metadata()
if err := rejectDuplicateBaseDefinitions(cfg, metadata, sources); err != nil {
return nil, err
}
resolved := &pipelineResolutionMetadata{
configkitMetadata: metadata, configkitSources: sources,
publishDeclared: metadata.IsExplicit(configkit.Keys("publish")),
}
pathsByID := make(map[string]string, len(sources))
for index, source := range sources {
path := source.Path
if path == "" {
path = source.Label
}
pathsByID[source.ID] = path
resolved.sources = append(resolved.sources, path)
if index == 0 {
resolved.rootPath = path
} else if source.Layer == configkit.LayerBase {
resolved.imports = append(resolved.imports, path)
}
if source.Layer == configkit.LayerProfile {
resolved.selectedProfile = &pipelineProfileSelection{name: selectedPipelineProfileName(opts), source: "explicit", overlayPath: path}
}
}
detailed, err := pipelineMetadataPaths(cfg)
if err != nil {
return nil, err
}
for _, value := range detailed {
info, ok := metadata.Lookup(configkit.Keys(value.segments...))
sourceNames := []string{pipelineDefaultOwnershipSource}
if ok {
sourceNames = ownershipSources(info.Origin, pathsByID)
}
resolved.ownership = append(resolved.ownership, pipelineFieldOwnership{path: value.record.Path, sources: sourceNames})
}
return resolved, nil
}
func pipelineMetadataPaths(cfg *PipelineConfig) ([]projectedPipelineValue, error) {
effective, err := projectionDetailedRecords(projectEffectivePipeline(cfg, false))
if err != nil {
return nil, err
}
if cfg.Scriptorium == nil || len(cfg.Scriptorium.ArtifactFamilies) == 0 {
return effective, nil
}
families, err := projectionDetailedRecords(map[string]any{"scriptorium": map[string]any{"artifact_families": cfg.Scriptorium.ArtifactFamilies}})
if err != nil {
return nil, err
}
return append(effective, families...), nil
}
func ownershipSources(origin configkit.Origin, pathsByID map[string]string) []string {
if origin.Layer == configkit.LayerDefault || (origin.Layer == configkit.LayerResolution && origin.Operation == configkit.ChangeDerivedDefault) {
return []string{pipelineDefaultOwnershipSource}
}
if origin.Layer == configkit.LayerResolution {
return []string{"resolution"}
}
seen := map[string]struct{}{}
var sources []string
for _, location := range origin.Locations {
value := pathsByID[location.SourceID]
if value == "" {
value = location.SourceID
}
if _, ok := seen[value]; !ok {
seen[value] = struct{}{}
sources = append(sources, value)
}
}
if len(sources) == 0 {
return []string{pipelineDefaultOwnershipSource}
}
return sources
}
func rejectDuplicateBaseDefinitions(cfg *PipelineConfig, metadata configkit.Metadata, sources []configkit.SourceInfo) error {
pathsByID := make(map[string]string, len(sources))
for _, source := range sources {
pathsByID[source.ID] = source.Path
}
var declarations []struct {
kind string
key string
path configkit.Path
}
if cfg.Scriptorium != nil {
for key := range cfg.Scriptorium.Artifacts {
declarations = append(declarations, struct {
kind, key string
path configkit.Path
}{"artifact", key, configkit.Keys("scriptorium", "artifacts", key)})
}
for key := range cfg.Scriptorium.ArtifactFamilies {
declarations = append(declarations, struct {
kind, key string
path configkit.Path
}{"artifact family", key, configkit.Keys("scriptorium", "artifact_families", key)})
}
}
sort.Slice(declarations, func(i, j int) bool { return declarations[i].path.String() < declarations[j].path.String() })
for _, declaration := range declarations {
seen := map[string]struct{}{}
for _, change := range metadata.History(declaration.path) {
if change.Origin.Layer != configkit.LayerBase {
continue
}
for _, location := range change.Origin.Locations {
seen[location.SourceID] = struct{}{}
}
}
if len(seen) > 1 {
claims := make([]string, 0, len(seen))
for sourceID := range seen {
claims = append(claims, pathsByID[sourceID])
}
sort.Strings(claims)
return fmt.Errorf("pipeline Scriptorium %s %q is declared by multiple base files: %s", declaration.kind, declaration.key, strings.Join(claims, ", "))
}
}
return nil
}
func intPtr(value int) *int { return &value }
func float64Ptr(value float64) *float64 { return &value }
func boolPtr(value bool) *bool { return &value }

View File

@@ -0,0 +1,102 @@
package config
import (
"os"
"path/filepath"
"strings"
"testing"
)
func TestPipelineConfigKitImportsProfileAndBaseSelection(t *testing.T) {
dir := t.TempDir()
writePipelineFixture(t, dir, "pipeline.yml", `configkit:
imports:
- path: config/workspace.yml
- path: config/artifact.yml
whisperx:
transcribe_url: https://transcription.example.test/transcribe
`)
writePipelineFixture(t, dir, "config/workspace.yml", "workspace:\n root: /srv/base\n")
writePipelineFixture(t, dir, "config/artifact.yml", `scriptorium:
artifacts:
recap:
enabled: false
output_path: artifacts/recap.md
`)
writePipelineFixture(t, dir, "profiles/testing.yml", "workspace:\n root: /srv/testing\n")
writePipelineFixture(t, dir, "profiles/unselected.yml", "unknown: [\n")
root := filepath.Join(dir, "pipeline.yml")
for _, profile := range []*string{nil, stringPointer(""), stringPointer("default")} {
cfg, err := LoadPipelineWithOptions(root, PipelineLoadOptions{Profile: profile})
if err != nil {
t.Fatalf("load base selection %#v: %v", profile, err)
}
if cfg.Workspace.Root != "/srv/base" {
t.Fatalf("base workspace = %q", cfg.Workspace.Root)
}
}
selected := "testing"
cfg, err := LoadPipelineWithOptions(root, PipelineLoadOptions{Profile: &selected})
if err != nil {
t.Fatal(err)
}
if cfg.Workspace.Root != "/srv/testing" || len(EffectivePipelineImports(cfg)) != 2 {
t.Fatalf("profiled pipeline = %#v, imports=%#v", cfg.Workspace, EffectivePipelineImports(cfg))
}
profile, ok := SelectedPipelineProfile(cfg)
if !ok || profile.Name != "testing" || profile.OverlayPath != filepath.Join(dir, "profiles", "testing.yml") {
t.Fatalf("selected profile = %#v, %v", profile, ok)
}
}
func TestPipelineConfigKitRejectsDuplicateBaseDeclarations(t *testing.T) {
for _, test := range []struct {
name, first, second string
}{
{name: "artifact", first: "scriptorium:\n artifacts:\n recap:\n prompt_id: recap\n", second: "scriptorium:\n artifacts:\n recap:\n output_path: artifacts/recap.md\n"},
{name: "artifact family", first: "scriptorium:\n artifact_families:\n recap:\n for_each: party.characters\n", second: "scriptorium:\n artifact_families:\n recap:\n output_path_pattern: artifacts/{character_id}.md\n"},
} {
t.Run(test.name, func(t *testing.T) {
dir := t.TempDir()
writePipelineFixture(t, dir, "pipeline.yml", `configkit:
imports:
- path: first.yml
- path: second.yml
whisperx:
transcribe_url: https://transcription.example.test/transcribe
`)
writePipelineFixture(t, dir, "first.yml", test.first)
writePipelineFixture(t, dir, "second.yml", test.second)
_, err := LoadPipeline(filepath.Join(dir, "pipeline.yml"))
if err == nil || !strings.Contains(err.Error(), "declared by multiple base files") ||
!strings.Contains(err.Error(), "first.yml") || !strings.Contains(err.Error(), "second.yml") {
t.Fatalf("duplicate declaration error = %v", err)
}
})
}
}
func TestPipelineConfigKitRejectsUnknownDirectiveAlias(t *testing.T) {
dir := t.TempDir()
path := writePipelineFixture(t, dir, "pipeline.yml", "composition:\n imports: []\n")
_, err := LoadPipeline(path)
if err == nil || !strings.Contains(err.Error(), "decode.unknown_field") {
t.Fatalf("composition alias error = %v", err)
}
}
func writePipelineFixture(t *testing.T, root, relative, contents string) string {
t.Helper()
path := filepath.Join(root, filepath.FromSlash(relative))
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, []byte(contents), 0o644); err != nil {
t.Fatal(err)
}
return path
}
func stringPointer(value string) *string { return &value }

View File

@@ -1,479 +0,0 @@
package config
import (
"os"
"path/filepath"
"slices"
"strings"
"testing"
"github.com/google/go-cmp/cmp"
)
func TestLoadPipelineProfilesSelectDefaultAndExplicitOverlay(t *testing.T) {
dir := t.TempDir()
rootPath := writeProfilePipeline(t, dir, `composition:
imports: [conf.d/base.yml]
default_profile: production
profiles:
production:
overlay: profiles/production.yml
testing:
overlay: profiles/testing.yml
whisperx:
transcribe_url: https://transcription.example.com/transcribe
`)
importPath := writePipelineSource(t, dir, "conf.d/base.yml", `workspace:
root: /srv/base
audita:
modules: [base, shared]
`)
productionPath := writePipelineSource(t, dir, "profiles/production.yml", `workspace:
root: /srv/production
whisperx:
language: en
`)
testingPath := writePipelineSource(t, dir, "profiles/testing.yml", `workspace:
root: /srv/testing
whisperx:
language: fr
`)
production, err := LoadPipeline(rootPath)
if err != nil {
t.Fatal(err)
}
if production.Workspace.Root != "/srv/production" || production.WhisperX.Language != "en" {
t.Fatalf("default profile result = workspace=%q language=%q", production.Workspace.Root, production.WhisperX.Language)
}
assertSelectedPipelineProfile(t, production, "production", "default", productionPath)
if want := []string{absolutePath(t, rootPath), absolutePath(t, importPath), absolutePath(t, productionPath)}; !slices.Equal(production.resolution.sources, want) {
t.Fatalf("default sources = %#v, want %#v", production.resolution.sources, want)
}
assertPipelineFieldOwner(t, production, "workspace.root", absolutePath(t, productionPath))
assertPipelineFieldOwner(t, production, "audita.modules", absolutePath(t, importPath))
assertPipelineFieldOwner(t, production, "storage.backend", pipelineDefaultOwnershipSource)
profile := "testing"
testingCfg, err := LoadPipelineWithOptions(rootPath, PipelineLoadOptions{Profile: &profile})
if err != nil {
t.Fatal(err)
}
if testingCfg.Workspace.Root != "/srv/testing" || testingCfg.WhisperX.Language != "fr" {
t.Fatalf("explicit profile result = workspace=%q language=%q", testingCfg.Workspace.Root, testingCfg.WhisperX.Language)
}
assertSelectedPipelineProfile(t, testingCfg, "testing", "cli", testingPath)
}
func TestLoadPipelineProfileSelectionErrors(t *testing.T) {
for _, test := range []struct {
name string
root string
profile *string
want string
}{
{name: "profiles require selection", root: profileComposition("", "production"), want: "default_profile is omitted"},
{name: "unknown explicit", root: profileComposition("production", "production"), profile: profilePointer("unknown"), want: "not declared"},
{name: "stacked explicit", root: profileComposition("production", "production", "testing"), profile: profilePointer("production,testing"), want: "not declared"},
{name: "explicit empty", root: profileComposition("production", "production"), profile: profilePointer(""), want: "must be non-empty"},
{name: "explicit whitespace", root: profileComposition("production", "production"), profile: profilePointer(" production "), want: "surrounding whitespace"},
{name: "invalid default", root: profileComposition("unknown", "production"), want: "does not name a declared profile"},
{name: "empty default", root: profileComposition("EMPTY", "production"), want: "must be non-empty"},
{name: "empty profile name", root: `composition:
default_profile: production
profiles:
"":
overlay: production.yml
whisperx:
transcribe_url: https://transcription.example.com/transcribe
`, want: "profile name must be non-empty"},
} {
t.Run(test.name, func(t *testing.T) {
dir := t.TempDir()
root := test.root
if test.name == "empty default" {
root = strings.ReplaceAll(root, "default_profile: EMPTY", `default_profile: ""`)
}
rootPath := writeProfilePipeline(t, dir, root)
writePipelineSource(t, dir, "production.yml", "whisperx:\n language: en\n")
_, err := LoadPipelineWithOptions(rootPath, PipelineLoadOptions{Profile: test.profile})
if err == nil || !strings.Contains(strings.ToLower(err.Error()), strings.ToLower(test.want)) {
t.Fatalf("LoadPipelineWithOptions() error = %v, want containing %q", err, test.want)
}
})
}
t.Run("explicit against profile-free pipeline", func(t *testing.T) {
dir := t.TempDir()
rootPath := writeProfilePipeline(t, dir, testPipelineBaseYAML)
selected := "testing"
_, err := LoadPipelineWithOptions(rootPath, PipelineLoadOptions{Profile: &selected})
if err == nil || !strings.Contains(err.Error(), "declares no profiles") {
t.Fatalf("error = %v, want profile-free rejection", err)
}
})
}
func TestPipelineProfileNamesRejectControlCharacters(t *testing.T) {
if _, err := normalizePipelineProfileName("production\x00testing", "profile"); err == nil || !strings.Contains(err.Error(), "control characters") {
t.Fatalf("normalizePipelineProfileName() error = %v, want control-character rejection", err)
}
}
func TestLoadPipelineProfilesValidateEveryDeclaredOverlay(t *testing.T) {
for _, test := range []struct {
name string
path string
content string
setup func(*testing.T, string)
want string
}{
{name: "missing", path: "profiles/missing.yml", want: "open composition.profiles.testing.overlay"},
{name: "malformed", path: "profiles/testing.yml", content: "workspace: [\n", want: "decode yaml"},
{name: "duplicate keys", path: "profiles/testing.yml", content: "workspace:\n root: /one\n root: /two\n", want: "duplicate yaml key"},
{name: "trailing document", path: "profiles/testing.yml", content: "workspace:\n root: /one\n---\nworkspace:\n root: /two\n", want: "exactly one yaml document"},
{name: "nested composition", path: "profiles/testing.yml", content: "composition:\n imports: [nested.yml]\n", want: "only the root pipeline"},
{name: "inheritance", path: "profiles/testing.yml", content: "workspace:\n root: /testing\n", setup: func(t *testing.T, dir string) {
rootPath := filepath.Join(dir, "pipeline.yml")
data, err := os.ReadFile(rootPath)
if err != nil {
t.Fatal(err)
}
updated := strings.Replace(string(data), "testing:\n overlay:", "testing:\n extends: production\n overlay:", 1)
if err := os.WriteFile(rootPath, []byte(updated), 0o644); err != nil {
t.Fatal(err)
}
}, want: "only overlay is supported"},
{name: "traversal", path: "../testing.yml", want: "invalid"},
{name: "extension", path: "profiles/testing.json", content: "{}\n", want: ".yml or .yaml"},
{name: "directory", path: "profiles/testing.yml", setup: func(t *testing.T, dir string) {
if err := os.MkdirAll(filepath.Join(dir, "profiles/testing.yml"), 0o755); err != nil {
t.Fatal(err)
}
}, want: "is a directory"},
} {
t.Run(test.name, func(t *testing.T) {
dir := t.TempDir()
rootPath := writeProfilePipeline(t, dir, `composition:
default_profile: production
profiles:
production:
overlay: profiles/production.yml
testing:
overlay: `+test.path+`
whisperx:
transcribe_url: https://transcription.example.com/transcribe
`)
writePipelineSource(t, dir, "profiles/production.yml", "whisperx:\n language: en\n")
if test.content != "" {
writePipelineSource(t, dir, test.path, test.content)
}
if test.setup != nil {
test.setup(t, dir)
}
_, err := LoadPipeline(rootPath)
if err == nil || !strings.Contains(strings.ToLower(err.Error()), strings.ToLower(test.want)) {
t.Fatalf("LoadPipeline() error = %v, want containing %q", err, test.want)
}
})
}
}
func TestLoadPipelineProfileOverlaySemantics(t *testing.T) {
dir := t.TempDir()
rootPath := writeProfilePipeline(t, dir, `composition:
default_profile: testing
profiles:
testing:
overlay: testing.yml
workspace:
root: /base
cleanup_after_publish: true
whisperx:
transcribe_url: https://transcription.example.com/transcribe
retries: 3
audita:
modules: [base, shared]
scriptorium:
artifacts:
session_recap:
enabled: true
prompt_id: dnd.session_recap
output_path: artifacts/session_recap.md
`)
writePipelineSource(t, dir, "testing.yml", `workspace:
cleanup_after_publish: false
whisperx:
retries: 0
audita:
modules: []
scriptorium:
artifacts:
session_recap:
enabled: false
experimental:
enabled: false
`)
cfg, err := LoadPipeline(rootPath)
if err != nil {
t.Fatal(err)
}
if cfg.Workspace.Root != "/base" || cfg.Workspace.CleanupAfterPublish || cfg.WhisperX.Retries == nil || *cfg.WhisperX.Retries != 0 {
t.Fatalf("recursive/zero overlay = workspace=%#v retries=%#v", cfg.Workspace, cfg.WhisperX.Retries)
}
if cfg.Audita.Modules == nil || len(cfg.Audita.Modules) != 0 {
t.Fatalf("list replacement = %#v, want explicit empty list", cfg.Audita.Modules)
}
if cfg.Scriptorium == nil || cfg.Scriptorium.Artifacts["session_recap"].Enabled || len(cfg.Scriptorium.Artifacts) != 2 {
t.Fatalf("keyed artifact overlay = %#v", cfg.Scriptorium)
}
}
func TestLoadPipelineProfileRejectsNullAndKindChanges(t *testing.T) {
for _, test := range []struct {
name string
overlay string
want string
}{
{name: "null deletion", overlay: "workspace:\n root: null\n", want: "null cannot delete"},
{name: "mapping scalar", overlay: "workspace: /other\n", want: "kind change"},
{name: "list mapping", overlay: "audita:\n modules:\n testing: true\n", want: "kind change"},
} {
t.Run(test.name, func(t *testing.T) {
dir := t.TempDir()
rootPath := writeProfilePipeline(t, dir, `composition:
default_profile: testing
profiles:
testing:
overlay: testing.yml
workspace:
root: /base
audita:
modules: [base]
whisperx:
transcribe_url: https://transcription.example.com/transcribe
`)
writePipelineSource(t, dir, "testing.yml", test.overlay)
_, err := LoadPipeline(rootPath)
if err == nil || !strings.Contains(strings.ToLower(err.Error()), strings.ToLower(test.want)) {
t.Fatalf("LoadPipeline() error = %v, want containing %q", err, test.want)
}
})
}
}
func TestPipelineEffectiveDigestTracksNormalizedMeaning(t *testing.T) {
dir := t.TempDir()
monolithicPath := writePipelineSource(t, dir, "monolithic.yml", `workspace:
root: /srv/narratio
whisperx:
transcribe_url: https://transcription.example.com/transcribe
language: en
`)
composedPath := writePipelineSource(t, dir, "pipeline.yml", `composition:
imports: [workspace.yml]
whisperx:
transcribe_url: https://transcription.example.com/transcribe
language: en
`)
writePipelineSource(t, dir, "workspace.yml", "workspace:\n root: /srv/narratio\n")
monolithic, err := LoadPipeline(monolithicPath)
if err != nil {
t.Fatal(err)
}
composed, err := LoadPipeline(composedPath)
if err != nil {
t.Fatal(err)
}
if monolithic.resolution.effectiveDigest == "" || monolithic.resolution.effectiveDigest != composed.resolution.effectiveDigest {
t.Fatalf("equal effective results have digests %q and %q", monolithic.resolution.effectiveDigest, composed.resolution.effectiveDigest)
}
changedPath := writePipelineSource(t, dir, "changed.yml", `workspace:
root: /srv/narratio
whisperx:
transcribe_url: https://transcription.example.com/transcribe
language: fr
`)
changed, err := LoadPipeline(changedPath)
if err != nil {
t.Fatal(err)
}
if changed.resolution.effectiveDigest == monolithic.resolution.effectiveDigest {
t.Fatal("semantic value change retained effective digest")
}
t.Setenv("AWS_SECRET_ACCESS_KEY", "secret-one")
first, err := LoadPipeline(monolithicPath)
if err != nil {
t.Fatal(err)
}
t.Setenv("AWS_SECRET_ACCESS_KEY", "secret-two")
second, err := LoadPipeline(monolithicPath)
if err != nil {
t.Fatal(err)
}
if first.resolution.effectiveDigest != second.resolution.effectiveDigest || strings.Contains(first.resolution.effectiveDigest, "secret") {
t.Fatalf("environment secret affected or appeared in digest: %q / %q", first.resolution.effectiveDigest, second.resolution.effectiveDigest)
}
}
func TestEffectivePipelineValuesIgnoreEquivalentSourceLayout(t *testing.T) {
dir := t.TempDir()
monolithicPath := writePipelineSource(t, dir, "monolithic.yml", `workspace:
root: /srv/narratio
whisperx:
transcribe_url: https://transcription.example.com/transcribe
language: en
`)
composedPath := writePipelineSource(t, dir, "pipeline.yml", `composition:
imports: [workspace.yml]
whisperx:
language: en
transcribe_url: https://transcription.example.com/transcribe
`)
writePipelineSource(t, dir, "workspace.yml", "workspace:\n root: /srv/narratio\n")
monolithic, err := LoadPipeline(monolithicPath)
if err != nil {
t.Fatal(err)
}
composed, err := LoadPipeline(composedPath)
if err != nil {
t.Fatal(err)
}
monolithicValues, err := EffectivePipelineValues(monolithic)
if err != nil {
t.Fatal(err)
}
composedValues, err := EffectivePipelineValues(composed)
if err != nil {
t.Fatal(err)
}
if diff := cmp.Diff(monolithicValues, composedValues); diff != "" {
t.Fatalf("effective values differ (-monolithic +composed):\n%s", diff)
}
}
func TestPipelineEffectiveDigestExcludesProfileIdentity(t *testing.T) {
dir := t.TempDir()
rootPath := writeProfilePipeline(t, dir, profileComposition("production", "production", "testing"))
overlay := "workspace:\n root: /srv/narratio\n"
writePipelineSource(t, dir, "production.yml", overlay)
writePipelineSource(t, dir, "testing.yml", overlay)
production := "production"
productionCfg, err := LoadPipelineWithOptions(rootPath, PipelineLoadOptions{Profile: &production})
if err != nil {
t.Fatal(err)
}
testing := "testing"
testingCfg, err := LoadPipelineWithOptions(rootPath, PipelineLoadOptions{Profile: &testing})
if err != nil {
t.Fatal(err)
}
if productionCfg.resolution.effectiveDigest != testingCfg.resolution.effectiveDigest {
t.Fatalf("profile identity changed equal effective digests: %q / %q", productionCfg.resolution.effectiveDigest, testingCfg.resolution.effectiveDigest)
}
}
func TestLoadPipelineProfilePairResolvesIndependentEffectivePipelines(t *testing.T) {
dir := t.TempDir()
rootPath := writeProfilePipeline(t, dir, `composition:
imports: [base.yml]
default_profile: production
profiles:
production:
overlay: production.yml
testing:
overlay: testing.yml
whisperx:
transcribe_url: https://transcription.example.com/transcribe
`)
writePipelineSource(t, dir, "base.yml", "workspace:\n root: /srv/base\n")
writePipelineSource(t, dir, "production.yml", "workspace:\n root: /srv/production\nwhisperx:\n language: en\n")
writePipelineSource(t, dir, "testing.yml", "workspace:\n root: /srv/testing\nwhisperx:\n language: fr\n")
production, testing, err := LoadPipelineProfilePair(rootPath, "production", "testing")
if err != nil {
t.Fatal(err)
}
if production.Workspace.Root != "/srv/production" || production.WhisperX.Language != "en" {
t.Fatalf("production pair result = %#v", production)
}
if testing.Workspace.Root != "/srv/testing" || testing.WhisperX.Language != "fr" {
t.Fatalf("testing pair result = %#v", testing)
}
production.Workspace.Root = "/mutated-left"
if testing.Workspace.Root != "/srv/testing" {
t.Fatalf("right profile shared mutable state with left: %q", testing.Workspace.Root)
}
testingFirst, productionSecond, err := LoadPipelineProfilePair(rootPath, "testing", "production")
if err != nil {
t.Fatal(err)
}
if testingFirst.Workspace.Root != "/srv/testing" || productionSecond.Workspace.Root != "/srv/production" {
t.Fatalf("reversed profile pair = testing=%q production=%q", testingFirst.Workspace.Root, productionSecond.Workspace.Root)
}
if _, _, err := LoadPipelineProfilePair(rootPath, "production", "production"); err == nil || !strings.Contains(err.Error(), "must differ") {
t.Fatalf("equal profile pair error = %v, want selection rejection", err)
}
if _, _, err := LoadPipelineProfilePair(rootPath, "unknown", "testing"); err == nil || !strings.Contains(err.Error(), "not declared") {
t.Fatalf("unknown profile pair error = %v, want selection rejection", err)
}
}
func TestLoadWithSessionOptionsCarriesExplicitProfilePresence(t *testing.T) {
dir := t.TempDir()
rootPath := writeProfilePipeline(t, dir, profileComposition("production", "production", "testing"))
writePipelineSource(t, dir, "production.yml", "workspace:\n root: /production\n")
writePipelineSource(t, dir, "testing.yml", "workspace:\n root: /testing\n")
campaignPath := writePipelineSource(t, dir, "campaign.yml", "campaign_id: campaign\ninputs:\n speakers_file: speakers.yml\n autocorrect_file: autocorrect.yml\n glossary_file: glossary.yml\n party_file: party.yml\n")
writePipelineSource(t, dir, "party.yml", canonicalPartyFixture)
sessionPath := writePipelineSource(t, dir, "session.yml", "session_id: session\ncampaign: campaign\n")
selected := "testing"
cfg, err := LoadWithSessionOptions(rootPath, campaignPath, sessionPath, SessionLoadOptions{Profile: &selected})
if err != nil {
t.Fatal(err)
}
if cfg.Pipeline.Workspace.Root != "/testing" || cfg.Pipeline.resolution.selectedProfile.source != "cli" {
t.Fatalf("combined profile result = workspace=%q metadata=%#v", cfg.Pipeline.Workspace.Root, cfg.Pipeline.resolution.selectedProfile)
}
}
func writeProfilePipeline(t *testing.T, dir, content string) string {
t.Helper()
return writePipelineSource(t, dir, "pipeline.yml", content)
}
func profileComposition(defaultProfile string, profiles ...string) string {
var builder strings.Builder
builder.WriteString("composition:\n")
if defaultProfile != "" {
builder.WriteString(" default_profile: " + defaultProfile + "\n")
}
builder.WriteString(" profiles:\n")
for _, profile := range profiles {
builder.WriteString(" " + profile + ":\n overlay: " + profile + ".yml\n")
}
builder.WriteString("whisperx:\n transcribe_url: https://transcription.example.com/transcribe\n")
return builder.String()
}
func profilePointer(value string) *string {
return &value
}
func assertSelectedPipelineProfile(t *testing.T, cfg *PipelineConfig, name, source, overlayPath string) {
t.Helper()
if cfg == nil || cfg.resolution == nil || cfg.resolution.selectedProfile == nil {
t.Fatal("selected profile provenance is absent")
}
selection := cfg.resolution.selectedProfile
if selection.name != name || selection.source != source || selection.overlayPath != absolutePath(t, overlayPath) {
t.Fatalf("selected profile = %#v, want name=%q source=%q overlay=%q", selection, name, source, absolutePath(t, overlayPath))
}
}

View File

@@ -129,7 +129,7 @@ func TestScriptoriumLoadAndValidate(t *testing.T) {
source: narratio.transcript.polished
artifact: session_recap
`,
wantLoadErr: "field artifact not found",
wantLoadErr: "decode.unknown_field",
},
{
name: "canonical previous-session source is accepted",

View File

@@ -540,7 +540,7 @@ publish:
`
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML)
_, err := Load(pipelinePath, sessionPath)
if err == nil || !strings.Contains(err.Error(), "strict decode failed") || !strings.Contains(err.Error(), "field locks not found") {
if err == nil || !strings.Contains(err.Error(), "decode.unknown_field") {
t.Fatalf("Load() error = %v, want retired locks field rejected", err)
}
}

View File

@@ -57,12 +57,14 @@ func TestTrimLoadAndValidate(t *testing.T) {
},
},
{
name: "enabled omitted defaults enabled",
name: "explicit null remains absent",
trimYAML: `trim:
`,
assert: func(t *testing.T, cfg *Config) {
t.Helper()
assertDefaultTrimConfig(t, cfg)
if cfg.Pipeline.Trim != nil {
t.Fatalf("trim config = %#v, want nil", cfg.Pipeline.Trim)
}
},
},
{

View File

@@ -60,12 +60,7 @@ func validatePublish(cfg *PublishConfig, scriptorium *ScriptoriumConfig, notariu
}
dest := strings.TrimSpace(item.Dest)
if dest == "" {
derivedDest, err := artifactpolicy.ResolvePublishedDestinationWithExtractions(source, "", configuredOutputs, extractionOutputs)
if err != nil {
return fmt.Errorf("%s.dest is required when destination cannot be derived from %q: %w", prefix, source, err)
}
dest = derivedDest
cfg.Outputs[i].Dest = derivedDest
return fmt.Errorf("%s.dest is required when destination cannot be derived from %q", prefix, source)
}
normalizedDest, err := pathsafe.NormalizeRelativeDestination(dest)
if err != nil {