diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index d407838..c4069fe 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -159,7 +159,7 @@ failing tests, stale documentation, or unmet exit criteria. ## Stage 1 — Presence-Aware Composition Engine -**Status: Pending** +**Status: Completed** ### Goal diff --git a/internal/config/composition.go b/internal/config/composition.go new file mode 100644 index 0000000..2a8ed80 --- /dev/null +++ b/internal/config/composition.go @@ -0,0 +1,657 @@ +package config + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "sort" + "strconv" + "strings" + + "gopkg.in/yaml.v3" +) + +// compositionDocument is the presence-aware representation used while +// assembling pipeline configuration sources. It deliberately models YAML +// mechanics rather than duplicating PipelineConfig's field schema. +type compositionDocument struct { + root *compositionNode + sources []string +} + +type compositionNode struct { + kind yaml.Kind + tag string + value string + path string + sources []string + line int + column int + fields []compositionField + items []*compositionNode +} + +type compositionField struct { + key string + value *compositionNode + order int + line int + column int +} + +// compositionValueRecord is a deterministic semantic leaf projection. Lists +// are atomic configuration values, while mappings are traversed recursively. +type compositionValueRecord struct { + Path string + Kind yaml.Kind + Value string + Sources []string +} + +// parseCompositionDocument parses exactly one YAML mapping while retaining +// source ownership, explicit zero values, and declaration order. +func parseCompositionDocument(source string, reader io.Reader) (*compositionDocument, error) { + if strings.TrimSpace(source) == "" { + return nil, fmt.Errorf("configuration source identity is required") + } + if reader == nil { + return nil, fmt.Errorf("configuration source %q: reader is nil", source) + } + + decoder := yaml.NewDecoder(reader) + var document yaml.Node + if err := decoder.Decode(&document); err != nil { + if err == io.EOF { + return nil, fmt.Errorf("configuration source %q: document is empty", source) + } + return nil, fmt.Errorf("configuration source %q: decode YAML: %w", source, err) + } + var trailing yaml.Node + if err := decoder.Decode(&trailing); err == nil { + return nil, fmt.Errorf("configuration source %q: must contain exactly one YAML document", source) + } else if err != io.EOF { + return nil, fmt.Errorf("configuration source %q: decode trailing YAML: %w", source, err) + } + if document.Kind != yaml.DocumentNode || len(document.Content) != 1 { + return nil, fmt.Errorf("configuration source %q: must contain exactly one YAML document", source) + } + if document.Content[0].Kind != yaml.MappingNode { + return nil, fmt.Errorf( + "configuration source %q: top-level document must be a mapping, got %s", + source, + yamlKindName(document.Content[0].Kind), + ) + } + + root, err := buildCompositionNode(document.Content[0], source, "") + if err != nil { + return nil, err + } + return &compositionDocument{root: root, sources: []string{source}}, nil +} + +func parseCompositionBytes(source string, data []byte) (*compositionDocument, error) { + return parseCompositionDocument(source, bytes.NewReader(data)) +} + +func buildCompositionNode(node *yaml.Node, source, path string) (*compositionNode, error) { + if node == nil { + return nil, fmt.Errorf("configuration source %q at %s: YAML node is nil", source, displayCompositionPath(path)) + } + if node.Kind == yaml.AliasNode { + return nil, compositionNodeError(source, path, node, "YAML aliases are not supported because source ownership would be ambiguous") + } + result := &compositionNode{ + kind: node.Kind, tag: node.Tag, value: node.Value, path: path, + sources: []string{source}, line: node.Line, column: node.Column, + } + + switch node.Kind { + case yaml.MappingNode: + if len(node.Content)%2 != 0 { + return nil, compositionNodeError(source, path, node, "mapping has an incomplete key/value pair") + } + seen := make(map[string]*yaml.Node, len(node.Content)/2) + for index := 0; index < len(node.Content); index += 2 { + keyNode := node.Content[index] + valueNode := node.Content[index+1] + if keyNode.Kind == yaml.AliasNode { + return nil, compositionNodeError(source, path, keyNode, "YAML aliases are not supported because source ownership would be ambiguous") + } + if valueNode.Kind == yaml.AliasNode { + return nil, compositionNodeError(source, appendCompositionPath(path, keyNode.Value), valueNode, "YAML aliases are not supported because source ownership would be ambiguous") + } + if keyNode.Kind != yaml.ScalarNode || keyNode.Tag != "!!str" { + return nil, compositionNodeError(source, path, keyNode, "mapping keys must be strings") + } + key := keyNode.Value + fieldPath := appendCompositionPath(path, key) + if prior, duplicate := seen[key]; duplicate { + return nil, fmt.Errorf( + "configuration source %q at %s: duplicate YAML key %q (first declared at line %d, column %d; repeated at line %d, column %d)", + source, displayCompositionPath(fieldPath), key, + prior.Line, prior.Column, keyNode.Line, keyNode.Column, + ) + } + seen[key] = keyNode + child, err := buildCompositionNode(valueNode, source, fieldPath) + if err != nil { + return nil, err + } + result.fields = append(result.fields, compositionField{ + key: key, value: child, order: len(result.fields), + line: keyNode.Line, column: keyNode.Column, + }) + } + case yaml.SequenceNode: + for index, childNode := range node.Content { + childPath := fmt.Sprintf("%s[%d]", path, index) + child, err := buildCompositionNode(childNode, source, childPath) + if err != nil { + return nil, err + } + result.items = append(result.items, child) + } + case yaml.ScalarNode: + // Scalar tag and lexical value retain distinctions such as explicit + // false, zero, an empty string, and null until final strict decoding. + default: + return nil, compositionNodeError(source, path, node, fmt.Sprintf("unsupported YAML node kind %s", yamlKindName(node.Kind))) + } + return result, nil +} + +// mergeAdditiveComposition recursively combines disjoint mappings. A scalar, +// list, or final keyed value may have only one base owner, regardless of +// whether duplicate values happen to be equal. +func mergeAdditiveComposition(base, incoming *compositionDocument) (*compositionDocument, error) { + if err := validateCompositionDocument(base, "base"); err != nil { + return nil, err + } + if err := validateCompositionDocument(incoming, "incoming"); err != nil { + return nil, err + } + merged, err := mergeAdditiveNodes(cloneCompositionNode(base.root), incoming.root) + if err != nil { + return nil, err + } + return &compositionDocument{ + root: merged, + sources: appendUniqueStrings( + append([]string(nil), base.sources...), incoming.sources..., + ), + }, nil +} + +func mergeAdditiveNodes(base, incoming *compositionNode) (*compositionNode, error) { + if base.kind != yaml.MappingNode || incoming.kind != yaml.MappingNode { + return nil, newCompositionConflict("additive base merge", base.path, base, incoming) + } + base.sources = appendUniqueStrings(base.sources, incoming.sources...) + for _, incomingField := range incoming.fields { + index := compositionFieldIndex(base.fields, incomingField.key) + if index < 0 { + field := cloneCompositionField(incomingField) + field.order = len(base.fields) + base.fields = append(base.fields, field) + continue + } + baseValue := base.fields[index].value + incomingValue := incomingField.value + if baseValue.kind == yaml.MappingNode && incomingValue.kind == yaml.MappingNode { + if len(baseValue.fields) == 0 || len(incomingValue.fields) == 0 { + return nil, newCompositionConflict("additive base merge", incomingValue.path, baseValue, incomingValue) + } + merged, err := mergeAdditiveNodes(baseValue, incomingValue) + if err != nil { + return nil, err + } + base.fields[index].value = merged + continue + } + return nil, newCompositionConflict("additive base merge", incomingValue.path, baseValue, incomingValue) + } + return base, nil +} + +// mergeOverlayComposition applies the sole overwrite layer. Mappings merge +// recursively; same-kind scalars and lists replace; null and kind changes are +// rejected. +func mergeOverlayComposition(base, overlay *compositionDocument) (*compositionDocument, error) { + if err := validateCompositionDocument(base, "base"); err != nil { + return nil, err + } + if err := validateCompositionDocument(overlay, "overlay"); err != nil { + return nil, err + } + if null := firstNullCompositionNode(overlay.root); null != nil { + return nil, fmt.Errorf( + "configuration overlay at %s from %s: null cannot delete an effective value", + displayCompositionPath(null.path), formatCompositionSources(null.sources), + ) + } + merged, err := mergeOverlayNodes(cloneCompositionNode(base.root), overlay.root) + if err != nil { + return nil, err + } + return &compositionDocument{ + root: merged, + sources: appendUniqueStrings( + append([]string(nil), base.sources...), overlay.sources..., + ), + }, nil +} + +func mergeOverlayNodes(base, overlay *compositionNode) (*compositionNode, error) { + if base.kind != yaml.MappingNode || overlay.kind != yaml.MappingNode { + return nil, newCompositionConflict("profile overlay", base.path, base, overlay) + } + base.sources = appendUniqueStrings(base.sources, overlay.sources...) + for _, overlayField := range overlay.fields { + index := compositionFieldIndex(base.fields, overlayField.key) + if index < 0 { + field := cloneCompositionField(overlayField) + field.order = len(base.fields) + base.fields = append(base.fields, field) + continue + } + baseValue := base.fields[index].value + overlayValue := overlayField.value + if baseValue.kind != overlayValue.kind { + return nil, newCompositionConflict("profile overlay kind change", overlayValue.path, baseValue, overlayValue) + } + if baseValue.kind == yaml.MappingNode { + merged, err := mergeOverlayNodes(baseValue, overlayValue) + if err != nil { + return nil, err + } + base.fields[index].value = merged + continue + } + base.fields[index].value = cloneCompositionNode(overlayValue) + } + return base, nil +} + +// canonicalYAML renders the effective mapping with sorted keys and normalized +// presentation while retaining sequence order and scalar YAML types. +func (document *compositionDocument) canonicalYAML() ([]byte, error) { + if err := validateCompositionDocument(document, "document"); err != nil { + return nil, err + } + root, err := compositionYAMLNode(document.root) + if err != nil { + return nil, err + } + var buffer bytes.Buffer + encoder := yaml.NewEncoder(&buffer) + encoder.SetIndent(2) + if err := encoder.Encode(root); err != nil { + return nil, fmt.Errorf("render effective configuration YAML: %w", err) + } + if err := encoder.Close(); err != nil { + return nil, fmt.Errorf("finish effective configuration YAML: %w", err) + } + return buffer.Bytes(), nil +} + +// canonicalDigestInput provides a deterministic, formatting-independent byte +// representation for later secret-free effective configuration digesting. +func (document *compositionDocument) canonicalDigestInput() ([]byte, error) { + if err := validateCompositionDocument(document, "document"); err != nil { + return nil, err + } + value, err := canonicalCompositionValue(document.root) + if err != nil { + return nil, err + } + data, err := json.Marshal(value) + if err != nil { + return nil, fmt.Errorf("serialize canonical configuration digest input: %w", err) + } + return append(data, '\n'), nil +} + +// semanticRecords returns sorted atomic values for future effective diff and +// source-report projections. A caller receives copies of all source slices. +func (document *compositionDocument) semanticRecords() ([]compositionValueRecord, error) { + if err := validateCompositionDocument(document, "document"); err != nil { + return nil, err + } + var records []compositionValueRecord + if err := appendCompositionRecords(document.root, &records); err != nil { + return nil, err + } + sort.Slice(records, func(i, j int) bool { return records[i].Path < records[j].Path }) + return records, nil +} + +func appendCompositionRecords(node *compositionNode, records *[]compositionValueRecord) error { + if node.kind == yaml.MappingNode && len(node.fields) > 0 { + for _, field := range node.fields { + if err := appendCompositionRecords(field.value, records); err != nil { + return err + } + } + return nil + } + value, err := canonicalCompositionValue(node) + if err != nil { + return err + } + encoded, err := json.Marshal(value) + if err != nil { + return fmt.Errorf("serialize configuration value at %s: %w", displayCompositionPath(node.path), err) + } + *records = append(*records, compositionValueRecord{ + Path: node.path, Kind: node.kind, Value: string(encoded), + Sources: append([]string(nil), node.sources...), + }) + return nil +} + +type canonicalCompositionField struct { + Key string `json:"key"` + Value any `json:"value"` +} + +type canonicalCompositionNode struct { + Kind string `json:"kind"` + Tag string `json:"tag,omitempty"` + Value any `json:"value,omitempty"` + Fields []canonicalCompositionField `json:"fields,omitempty"` + Items []any `json:"items,omitempty"` +} + +func canonicalCompositionValue(node *compositionNode) (any, error) { + switch node.kind { + case yaml.MappingNode: + fields := append([]compositionField(nil), node.fields...) + sort.Slice(fields, func(i, j int) bool { return fields[i].key < fields[j].key }) + result := canonicalCompositionNode{Kind: "mapping"} + if len(fields) == 0 { + result.Fields = []canonicalCompositionField{} + } + for _, field := range fields { + value, err := canonicalCompositionValue(field.value) + if err != nil { + return nil, err + } + result.Fields = append(result.Fields, canonicalCompositionField{Key: field.key, Value: value}) + } + return result, nil + case yaml.SequenceNode: + result := canonicalCompositionNode{Kind: "sequence", Items: make([]any, 0, len(node.items))} + for _, item := range node.items { + value, err := canonicalCompositionValue(item) + if err != nil { + return nil, err + } + result.Items = append(result.Items, value) + } + return result, nil + case yaml.ScalarNode: + value, err := canonicalScalarValue(node) + if err != nil { + return nil, err + } + return canonicalCompositionNode{Kind: "scalar", Tag: node.tag, Value: value}, nil + default: + return nil, fmt.Errorf("configuration at %s has unsupported YAML kind %s", displayCompositionPath(node.path), yamlKindName(node.kind)) + } +} + +func canonicalScalarValue(node *compositionNode) (any, error) { + raw := &yaml.Node{Kind: yaml.ScalarNode, Tag: node.tag, Value: node.value} + var value any + if err := raw.Decode(&value); err != nil { + return nil, fmt.Errorf("decode scalar at %s: %w", displayCompositionPath(node.path), err) + } + switch typed := value.(type) { + case nil, bool, string, int, int64, uint64, float64: + return typed, nil + default: + // yaml.v3 may decode timestamps or uncommon scalar tags into types that + // encoding/json can serialize deterministically. Preserve the resolved + // tag alongside the value in the containing canonical node. + return typed, nil + } +} + +func compositionYAMLNode(node *compositionNode) (*yaml.Node, error) { + switch node.kind { + case yaml.MappingNode: + result := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"} + fields := append([]compositionField(nil), node.fields...) + sort.Slice(fields, func(i, j int) bool { return fields[i].key < fields[j].key }) + for _, field := range fields { + value, err := compositionYAMLNode(field.value) + if err != nil { + return nil, err + } + result.Content = append(result.Content, + &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: field.key}, + value, + ) + } + return result, nil + case yaml.SequenceNode: + result := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"} + for _, item := range node.items { + value, err := compositionYAMLNode(item) + if err != nil { + return nil, err + } + result.Content = append(result.Content, value) + } + return result, nil + case yaml.ScalarNode: + return normalizedCompositionScalarNode(node) + default: + return nil, fmt.Errorf("configuration at %s has unsupported YAML kind %s", displayCompositionPath(node.path), yamlKindName(node.kind)) + } +} + +func normalizedCompositionScalarNode(node *compositionNode) (*yaml.Node, error) { + raw := &yaml.Node{Kind: yaml.ScalarNode, Tag: node.tag, Value: node.value} + var value any + if err := raw.Decode(&value); err != nil { + return nil, fmt.Errorf("normalize scalar at %s: %w", displayCompositionPath(node.path), err) + } + normalized := &yaml.Node{} + if err := normalized.Encode(value); err != nil { + return nil, fmt.Errorf("encode normalized scalar at %s: %w", displayCompositionPath(node.path), err) + } + if normalized.Kind != yaml.ScalarNode { + return nil, fmt.Errorf("normalize scalar at %s produced YAML kind %s", displayCompositionPath(node.path), yamlKindName(normalized.Kind)) + } + return normalized, nil +} + +func validateCompositionDocument(document *compositionDocument, role string) error { + if document == nil || document.root == nil { + return fmt.Errorf("configuration composition %s document is nil", role) + } + if document.root.kind != yaml.MappingNode { + return fmt.Errorf("configuration composition %s root must be a mapping", role) + } + return nil +} + +func firstNullCompositionNode(node *compositionNode) *compositionNode { + if node == nil { + return nil + } + if node.kind == yaml.ScalarNode && node.tag == "!!null" { + return node + } + for _, field := range node.fields { + if found := firstNullCompositionNode(field.value); found != nil { + return found + } + } + for _, item := range node.items { + if found := firstNullCompositionNode(item); found != nil { + return found + } + } + return nil +} + +func cloneCompositionNode(node *compositionNode) *compositionNode { + if node == nil { + return nil + } + clone := &compositionNode{ + kind: node.kind, tag: node.tag, value: node.value, path: node.path, + sources: append([]string(nil), node.sources...), line: node.line, column: node.column, + } + for _, field := range node.fields { + clone.fields = append(clone.fields, cloneCompositionField(field)) + } + for _, item := range node.items { + clone.items = append(clone.items, cloneCompositionNode(item)) + } + return clone +} + +func cloneCompositionField(field compositionField) compositionField { + return compositionField{ + key: field.key, value: cloneCompositionNode(field.value), order: field.order, + line: field.line, column: field.column, + } +} + +func compositionFieldIndex(fields []compositionField, key string) int { + for index := range fields { + if fields[index].key == key { + return index + } + } + return -1 +} + +func appendCompositionPath(parent, key string) string { + if isSimpleCompositionPathSegment(key) { + if parent == "" { + return key + } + return parent + "." + key + } + if parent == "" { + return "[" + strconv.Quote(key) + "]" + } + return parent + "[" + strconv.Quote(key) + "]" +} + +func isSimpleCompositionPathSegment(value string) bool { + if value == "" { + return false + } + for index, char := range value { + if (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') || char == '_' || (index > 0 && char >= '0' && char <= '9') || (index > 0 && char == '-') { + continue + } + return false + } + return true +} + +func displayCompositionPath(path string) string { + if path == "" { + return "" + } + return path +} + +func compositionNodeError(source, path string, node *yaml.Node, message string) error { + line, column := 0, 0 + if node != nil { + line, column = node.Line, node.Column + } + return fmt.Errorf( + "configuration source %q at %s (line %d, column %d): %s", + source, displayCompositionPath(path), line, column, message, + ) +} + +func newCompositionConflict(operation, path string, values ...*compositionNode) error { + var sources []string + var kinds []string + for _, value := range values { + if value == nil { + continue + } + sources = appendUniqueStrings(sources, compositionClaimSources(value)...) + kind := yamlKindName(value.kind) + if !containsString(kinds, kind) { + kinds = append(kinds, kind) + } + } + return fmt.Errorf( + "configuration %s conflict at %s: claimed by %s (YAML kinds: %s)", + operation, displayCompositionPath(path), formatCompositionSources(sources), strings.Join(kinds, ", "), + ) +} + +func compositionClaimSources(node *compositionNode) []string { + if node == nil { + return nil + } + sources := append([]string(nil), node.sources...) + for _, field := range node.fields { + sources = appendUniqueStrings(sources, compositionClaimSources(field.value)...) + } + for _, item := range node.items { + sources = appendUniqueStrings(sources, compositionClaimSources(item)...) + } + return sources +} + +func appendUniqueStrings(values []string, additions ...string) []string { + seen := make(map[string]struct{}, len(values)+len(additions)) + result := make([]string, 0, len(values)+len(additions)) + for _, value := range append(append([]string(nil), values...), additions...) { + if _, exists := seen[value]; exists { + continue + } + seen[value] = struct{}{} + result = append(result, value) + } + return result +} + +func containsString(values []string, target string) bool { + for _, value := range values { + if value == target { + return true + } + } + return false +} + +func formatCompositionSources(sources []string) string { + quoted := make([]string, 0, len(sources)) + for _, source := range sources { + quoted = append(quoted, strconv.Quote(source)) + } + return strings.Join(quoted, ", ") +} + +func yamlKindName(kind yaml.Kind) string { + switch kind { + case yaml.DocumentNode: + return "document" + case yaml.MappingNode: + return "mapping" + case yaml.SequenceNode: + return "sequence" + case yaml.ScalarNode: + return "scalar" + case yaml.AliasNode: + return "alias" + default: + return fmt.Sprintf("kind(%d)", kind) + } +} diff --git a/internal/config/composition_test.go b/internal/config/composition_test.go new file mode 100644 index 0000000..9c89ff2 --- /dev/null +++ b/internal/config/composition_test.go @@ -0,0 +1,356 @@ +package config + +import ( + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +func TestParseCompositionDocumentRetainsPresenceOwnershipAndDeclarationOrder(t *testing.T) { + document := mustParseComposition(t, "root.yml", `zeta: false +zero: 0 +empty_map: {} +empty_list: [] +nested: + value: "" +`) + + wantOrder := []string{"zeta", "zero", "empty_map", "empty_list", "nested"} + gotOrder := make([]string, 0, len(document.root.fields)) + for _, field := range document.root.fields { + gotOrder = append(gotOrder, field.key) + } + if !reflect.DeepEqual(gotOrder, wantOrder) { + t.Fatalf("declaration order = %#v, want %#v", gotOrder, wantOrder) + } + + tests := []struct { + path string + kind yaml.Kind + tag string + value string + }{ + {path: "zeta", kind: yaml.ScalarNode, tag: "!!bool", value: "false"}, + {path: "zero", kind: yaml.ScalarNode, tag: "!!int", value: "0"}, + {path: "empty_map", kind: yaml.MappingNode, tag: "!!map"}, + {path: "empty_list", kind: yaml.SequenceNode, tag: "!!seq"}, + {path: "nested.value", kind: yaml.ScalarNode, tag: "!!str", value: ""}, + } + for _, tt := range tests { + node := compositionNodeAtPath(t, document.root, tt.path) + if node.kind != tt.kind || node.tag != tt.tag || node.value != tt.value || !reflect.DeepEqual(node.sources, []string{"root.yml"}) { + t.Fatalf("node %s = kind=%v tag=%q value=%q sources=%#v", tt.path, node.kind, node.tag, node.value, node.sources) + } + } +} + +func TestParseCompositionDocumentRejectsAmbiguousOrMalformedYAML(t *testing.T) { + tests := []struct { + name string + yaml string + want string + }{ + {name: "duplicate top-level key", yaml: "value: 1\nvalue: 2\n", want: "duplicate YAML key"}, + {name: "duplicate nested key", yaml: "outer:\n value: 1\n value: 2\n", want: "outer.value"}, + {name: "alias", yaml: "base: &base\n value: 1\ncopy: *base\n", want: "aliases are not supported"}, + {name: "trailing document", yaml: "value: 1\n---\nvalue: 2\n", want: "exactly one YAML document"}, + {name: "top-level sequence", yaml: "- value\n", want: "top-level document must be a mapping"}, + {name: "non-string key", yaml: "1: value\n", want: "mapping keys must be strings"}, + {name: "malformed", yaml: "outer: [\n", want: "decode YAML"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := parseCompositionDocument("broken.yml", strings.NewReader(tt.yaml)) + if err == nil { + t.Fatal("parseCompositionDocument() error = nil") + } + if !strings.Contains(err.Error(), "broken.yml") || !strings.Contains(err.Error(), tt.want) { + t.Fatalf("error = %q, want source and %q", err, tt.want) + } + }) + } +} + +func TestMergeAdditiveCompositionJoinsOnlyDisjointMappings(t *testing.T) { + root := mustParseComposition(t, "pipeline.yml", `scriptorium: + artifacts: + recap: + enabled: true +zero: 0 +`) + imports := mustParseComposition(t, "conf.d/artifacts.yml", `scriptorium: + artifacts: + handout: + enabled: false +empty_list: [] +`) + + merged, err := mergeAdditiveComposition(root, imports) + if err != nil { + t.Fatalf("mergeAdditiveComposition() error = %v", err) + } + for _, path := range []string{ + "empty_list", "scriptorium.artifacts.handout.enabled", + "scriptorium.artifacts.recap.enabled", "zero", + } { + _ = compositionNodeAtPath(t, merged.root, path) + } + if got := compositionNodeAtPath(t, merged.root, "scriptorium.artifacts.handout.enabled").sources; !reflect.DeepEqual(got, []string{"conf.d/artifacts.yml"}) { + t.Fatalf("handout sources = %#v", got) + } + if got := compositionNodeAtPath(t, merged.root, "scriptorium.artifacts.recap.enabled").sources; !reflect.DeepEqual(got, []string{"pipeline.yml"}) { + t.Fatalf("recap sources = %#v", got) + } + + // Merge operations return a new document and retain the declared order in + // each input for source-aware diagnostics. + if len(root.root.fields) != 2 || len(imports.root.fields) != 2 { + t.Fatalf("merge mutated inputs: root=%d import=%d", len(root.root.fields), len(imports.root.fields)) + } +} + +func TestMergeAdditiveCompositionRejectsEveryDuplicateClass(t *testing.T) { + tests := []struct { + name string + baseYAML string + nextYAML string + path string + }{ + {name: "equal scalar", baseYAML: "value: true\n", nextYAML: "value: true\n", path: "value"}, + {name: "different scalar", baseYAML: "value: true\n", nextYAML: "value: false\n", path: "value"}, + {name: "atomic list", baseYAML: "values: [one]\n", nextYAML: "values: [two]\n", path: "values"}, + {name: "keyed entry", baseYAML: "items:\n shared:\n left: 1\n", nextYAML: "items:\n shared:\n left: 2\n", path: "items.shared.left"}, + {name: "kind conflict", baseYAML: "value:\n nested: true\n", nextYAML: "value: scalar\n", path: "value"}, + {name: "duplicate empty map", baseYAML: "value: {}\n", nextYAML: "value: {}\n", path: "value"}, + {name: "empty map then populated map", baseYAML: "value: {}\n", nextYAML: "value: {nested: true}\n", path: "value"}, + {name: "populated map then empty map", baseYAML: "value: {nested: true}\n", nextYAML: "value: {}\n", path: "value"}, + {name: "duplicate empty list", baseYAML: "value: []\n", nextYAML: "value: []\n", path: "value"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + base := mustParseComposition(t, "base.yml", tt.baseYAML) + next := mustParseComposition(t, "next.yml", tt.nextYAML) + _, err := mergeAdditiveComposition(base, next) + if err == nil { + t.Fatal("mergeAdditiveComposition() error = nil") + } + for _, want := range []string{tt.path, "base.yml", "next.yml"} { + if !strings.Contains(err.Error(), want) { + t.Fatalf("error = %q, want %q", err, want) + } + } + }) + } +} + +func TestMergeAdditiveCompositionReportsAllClaimingSources(t *testing.T) { + left := mustParseComposition(t, "left.yml", "group:\n left: 1\n") + right := mustParseComposition(t, "right.yml", "group:\n right: 2\n") + base, err := mergeAdditiveComposition(left, right) + if err != nil { + t.Fatal(err) + } + overlap := mustParseComposition(t, "overlap.yml", "group: scalar\n") + _, err = mergeAdditiveComposition(base, overlap) + if err == nil { + t.Fatal("mergeAdditiveComposition() error = nil") + } + for _, want := range []string{"group", "left.yml", "right.yml", "overlap.yml"} { + if !strings.Contains(err.Error(), want) { + t.Fatalf("error = %q, want %q", err, want) + } + } +} + +func TestMergeOverlayCompositionRecursesMapsAndReplacesAtomicValues(t *testing.T) { + base := mustParseComposition(t, "base.yml", `feature: + enabled: true + retries: 3 + values: [one, two] + inherited: kept +artifacts: + recap: + enabled: true +`) + overlay := mustParseComposition(t, "testing.yml", `feature: + enabled: false + retries: 0 + values: [] + added: present +artifacts: + handout: + enabled: false +`) + + merged, err := mergeOverlayComposition(base, overlay) + if err != nil { + t.Fatalf("mergeOverlayComposition() error = %v", err) + } + assertCompositionScalar(t, merged.root, "feature.enabled", "!!bool", "false", "testing.yml") + assertCompositionScalar(t, merged.root, "feature.retries", "!!int", "0", "testing.yml") + assertCompositionScalar(t, merged.root, "feature.inherited", "!!str", "kept", "base.yml") + assertCompositionScalar(t, merged.root, "feature.added", "!!str", "present", "testing.yml") + if values := compositionNodeAtPath(t, merged.root, "feature.values"); values.kind != yaml.SequenceNode || len(values.items) != 0 || !reflect.DeepEqual(values.sources, []string{"testing.yml"}) { + t.Fatalf("replaced list = %#v", values) + } + _ = compositionNodeAtPath(t, merged.root, "artifacts.recap.enabled") + _ = compositionNodeAtPath(t, merged.root, "artifacts.handout.enabled") +} + +func TestMergeOverlayCompositionRejectsKindChangesAndNullDeletion(t *testing.T) { + kindTests := []struct { + name string + baseYAML string + overlay string + }{ + {name: "map to scalar", baseYAML: "value: {nested: true}\n", overlay: "value: replacement\n"}, + {name: "scalar to map", baseYAML: "value: original\n", overlay: "value: {nested: true}\n"}, + {name: "list to scalar", baseYAML: "value: [one]\n", overlay: "value: replacement\n"}, + {name: "scalar to list", baseYAML: "value: original\n", overlay: "value: [one]\n"}, + } + for _, tt := range kindTests { + t.Run(tt.name, func(t *testing.T) { + _, err := mergeOverlayComposition( + mustParseComposition(t, "base.yml", tt.baseYAML), + mustParseComposition(t, "overlay.yml", tt.overlay), + ) + if err == nil || !strings.Contains(err.Error(), "value") || !strings.Contains(err.Error(), "kind change") { + t.Fatalf("error = %v, want value kind change", err) + } + }) + } + + for _, overlayYAML := range []string{"value: null\n", "value: ~\n", "nested:\n value:\n"} { + _, err := mergeOverlayComposition( + mustParseComposition(t, "base.yml", "value: original\nnested:\n value: original\n"), + mustParseComposition(t, "overlay.yml", overlayYAML), + ) + if err == nil || !strings.Contains(err.Error(), "null cannot delete") || !strings.Contains(err.Error(), "overlay.yml") { + t.Fatalf("error = %v, want source-qualified null rejection", err) + } + } +} + +func TestCompositionCanonicalOutputsAreDeterministic(t *testing.T) { + first := mustParseComposition(t, "first.yml", `zeta: 01 +alpha: + list: [true, false] + empty: {} +`) + second := mustParseComposition(t, "second.yml", `alpha: + empty: {} + list: + - true + - false +zeta: 1 +`) + + firstYAML, err := first.canonicalYAML() + if err != nil { + t.Fatal(err) + } + secondYAML, err := second.canonicalYAML() + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(firstYAML, secondYAML) { + t.Fatalf("canonical YAML differs:\n%s\n---\n%s", firstYAML, secondYAML) + } + firstDigest, err := first.canonicalDigestInput() + if err != nil { + t.Fatal(err) + } + secondDigest, err := second.canonicalDigestInput() + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(firstDigest, secondDigest) { + t.Fatalf("digest input differs:\n%s\n---\n%s", firstDigest, secondDigest) + } + + left := mustParseComposition(t, "left.yml", "zeta: 1\n") + right := mustParseComposition(t, "right.yml", "alpha: 2\n") + leftRight, err := mergeAdditiveComposition(left, right) + if err != nil { + t.Fatal(err) + } + rightLeft, err := mergeAdditiveComposition(right, left) + if err != nil { + t.Fatal(err) + } + want, _ := leftRight.canonicalDigestInput() + got, _ := rightLeft.canonicalDigestInput() + if !reflect.DeepEqual(got, want) { + t.Fatalf("source traversal changed semantic digest input: got %s want %s", got, want) + } + wantYAML, _ := leftRight.canonicalYAML() + gotYAML, _ := rightLeft.canonicalYAML() + if !reflect.DeepEqual(gotYAML, wantYAML) { + t.Fatalf("source traversal changed canonical YAML: got %s want %s", gotYAML, wantYAML) + } + + records, err := first.semanticRecords() + if err != nil { + t.Fatal(err) + } + paths := make([]string, 0, len(records)) + for _, record := range records { + paths = append(paths, record.Path) + } + if wantPaths := []string{"alpha.empty", "alpha.list", "zeta"}; !reflect.DeepEqual(paths, wantPaths) { + t.Fatalf("semantic record paths = %#v, want %#v", paths, wantPaths) + } +} + +func TestPipelineCompositionEnvelopeIsNotPublicYet(t *testing.T) { + path := filepath.Join(t.TempDir(), "pipeline.yml") + if err := os.WriteFile(path, []byte(`composition: + imports: [] +whisperx: + transcribe_url: https://transcription.example.com/transcribe +`), 0o644); err != nil { + t.Fatal(err) + } + _, err := LoadPipeline(path) + if err == nil || !strings.Contains(err.Error(), "field composition not found") { + t.Fatalf("LoadPipeline() error = %v, want strict public-schema rejection", err) + } +} + +func mustParseComposition(t *testing.T, source, input string) *compositionDocument { + t.Helper() + document, err := parseCompositionBytes(source, []byte(input)) + if err != nil { + t.Fatalf("parseCompositionBytes(%q) error = %v", source, err) + } + return document +} + +func compositionNodeAtPath(t *testing.T, root *compositionNode, path string) *compositionNode { + t.Helper() + node := root + for _, segment := range strings.Split(path, ".") { + if node == nil || node.kind != yaml.MappingNode { + t.Fatalf("path %q reached non-mapping at %q", path, segment) + } + index := compositionFieldIndex(node.fields, segment) + if index < 0 { + t.Fatalf("path %q missing segment %q", path, segment) + } + node = node.fields[index].value + } + return node +} + +func assertCompositionScalar(t *testing.T, root *compositionNode, path, tag, value, source string) { + t.Helper() + node := compositionNodeAtPath(t, root, path) + if node.kind != yaml.ScalarNode || node.tag != tag || node.value != value || !reflect.DeepEqual(node.sources, []string{source}) { + t.Fatalf("%s = kind=%s tag=%q value=%q sources=%#v", path, yamlKindName(node.kind), node.tag, node.value, node.sources) + } +}