Preserve null collections in effective YAML

internal/config/effective_pipeline.go

@@ -1,9 +1,11 @@
 package config
 
 import (
+	"encoding/json"
 	"fmt"
 	"path/filepath"
 	"sort"
+	"strconv"
 	"strings"
 
 	"gopkg.in/yaml.v3"
@@ -151,13 +153,72 @@ func MarshalEffectivePipeline(cfg *PipelineConfig) ([]byte, error) {
 		return nil, fmt.Errorf("pipeline config is required")
 	}
 
-	data, err := yaml.Marshal(projectEffectivePipeline(cfg, true))
+	value, err := projectionValueTree(projectEffectivePipeline(cfg, true))
+	if err != nil {
+		return nil, err
+	}
+	node, err := effectiveYAMLNode(value)
+	if err != nil {
+		return nil, fmt.Errorf("serialize effective pipeline: %w", err)
+	}
+	data, err := yaml.Marshal(node)
 	if err != nil {
 		return nil, fmt.Errorf("serialize effective pipeline: %w", err)
 	}
 	return data, nil
 }
 
+func effectiveYAMLNode(value any) (*yaml.Node, error) {
+	switch value := value.(type) {
+	case nil:
+		return &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!null", Value: "null"}, nil
+	case bool:
+		return &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!bool", Value: strconv.FormatBool(value)}, nil
+	case string:
+		return &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: value}, nil
+	case json.Number:
+		tag := "!!int"
+		if strings.ContainsAny(value.String(), ".eE") {
+			tag = "!!float"
+		}
+		return &yaml.Node{Kind: yaml.ScalarNode, Tag: tag, Value: value.String()}, nil
+	case []any:
+		node := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
+		if len(value) == 0 {
+			node.Style = yaml.FlowStyle
+		}
+		for _, item := range value {
+			child, err := effectiveYAMLNode(item)
+			if err != nil {
+				return nil, err
+			}
+			node.Content = append(node.Content, child)
+		}
+		return node, nil
+	case map[string]any:
+		node := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
+		if len(value) == 0 {
+			node.Style = yaml.FlowStyle
+		}
+		keys := make([]string, 0, len(value))
+		for key := range value {
+			keys = append(keys, key)
+		}
+		sort.Strings(keys)
+		for _, key := range keys {
+			child, err := effectiveYAMLNode(value[key])
+			if err != nil {
+				return nil, err
+			}
+			node.Content = append(node.Content,
+				&yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: key}, child)
+		}
+		return node, nil
+	default:
+		return nil, fmt.Errorf("unsupported projected value type %T", value)
+	}
+}
+
 func effectivePipelineOwners(cfg *PipelineConfig) map[string][]string {
 	owners := make(map[string][]string)
 	if cfg == nil || cfg.resolution == nil {

internal/config/effective_projection.go

@@ -74,6 +74,20 @@ func projectionJSON(projection effectivePipelineProjection) ([]byte, error) {
 	return data, nil
 }
 
+func projectionValueTree(value any) (any, error) {
+	data, err := json.Marshal(value)
+	if err != nil {
+		return nil, fmt.Errorf("encode typed pipeline projection: %w", err)
+	}
+	decoder := json.NewDecoder(bytes.NewReader(data))
+	decoder.UseNumber()
+	var root any
+	if err := decoder.Decode(&root); err != nil {
+		return nil, fmt.Errorf("decode effective pipeline projection: %w", err)
+	}
+	return root, nil
+}
+
 func projectionValueRecords(projection effectivePipelineProjection) ([]EffectivePipelineValueRecord, error) {
 	detailed, err := projectionDetailedRecords(projection)
 	if err != nil {
@@ -87,15 +101,13 @@ func projectionValueRecords(projection effectivePipelineProjection) ([]Effective
 }
 
 func projectionDetailedRecords(value any) ([]projectedPipelineValue, error) {
-	data, err := json.Marshal(value)
+	decoded, err := projectionValueTree(value)
 	if err != nil {
-		return nil, fmt.Errorf("encode typed pipeline projection: %w", err)
+		return nil, err
 	}
-	decoder := json.NewDecoder(bytes.NewReader(data))
-	decoder.UseNumber()
-	var root map[string]any
-	if err := decoder.Decode(&root); err != nil {
-		return nil, fmt.Errorf("decode effective pipeline projection: %w", err)
+	root, ok := decoded.(map[string]any)
+	if !ok {
+		return nil, fmt.Errorf("effective pipeline projection must be a mapping")
 	}
 	var records []projectedPipelineValue
 	appendProjectionRecords(&records, nil, root)

internal/config/load_validate_test.go

@@ -159,15 +159,44 @@ storage:
 }
 
 func TestEffectivePipelineValuesPreserveNilEmptyAndExactIntegers(t *testing.T) {
-	base := &PipelineConfig{Audita: AuditaConfig{Modules: nil}}
+	base := &PipelineConfig{
+		Audita:      AuditaConfig{Modules: nil},
+		Scriptorium: &ScriptoriumConfig{Artifacts: nil},
+		resolution:  &pipelineResolutionMetadata{},
+	}
 	nilValues, err := EffectivePipelineValues(base)
 	if err != nil {
 		t.Fatal(err)
 	}
+	nilYAML, err := MarshalEffectivePipeline(base)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(string(nilYAML), "modules: null\n") || !strings.Contains(string(nilYAML), "artifacts: null\n") {
+		t.Fatalf("effective YAML lost null collections: %q", nilYAML)
+	}
+	if err := recomputePipelineEffectiveDigest(base); err != nil {
+		t.Fatal(err)
+	}
+	nilDigest := EffectivePipelineDigest(base)
 	base.Audita.Modules = []string{}
-	base.Scriptorium = &ScriptoriumConfig{Artifacts: map[string]ScriptoriumArtifactConfig{
+	base.Scriptorium.Artifacts = map[string]ScriptoriumArtifactConfig{}
+	emptyYAML, err := MarshalEffectivePipeline(base)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(string(emptyYAML), "modules: []\n") || !strings.Contains(string(emptyYAML), "artifacts: {}\n") {
+		t.Fatalf("effective YAML lost empty collections: %q", emptyYAML)
+	}
+	if err := recomputePipelineEffectiveDigest(base); err != nil {
+		t.Fatal(err)
+	}
+	if emptyDigest := EffectivePipelineDigest(base); emptyDigest == nilDigest {
+		t.Fatalf("null and empty collections have equal digest %q", emptyDigest)
+	}
+	base.Scriptorium.Artifacts = map[string]ScriptoriumArtifactConfig{
 		"recap": {Vars: map[string]any{"exact": int64(9007199254740993)}},
-	}}
+	}
 	emptyValues, err := EffectivePipelineValues(base)
 	if err != nil {
 		t.Fatal(err)