Preserve null collections in effective YAML

docs/roadmap/remediation.md

@@ -128,7 +128,13 @@ valid overlay remains accepted.
 
 ## 4. Preserve null versus empty collections in effective YAML
 
-Status: Open — P2
+Status: Resolved — P2
+
+Resolution: Effective YAML is now emitted from the credential-safe JSON value
+projection through an output-only YAML-node conversion. Explicit nulls, empty maps,
+empty slices, booleans, strings, and exact JSON numbers retain their semantic types.
+Public show/diff tests cover null and empty maps and slices, while digest,
+credential-redaction, deterministic output, and exact-integer coverage remain intact.
 
 **Evidence:** `internal/config/effective_pipeline.go:154` marshals the typed
 projection directly with `yaml.Marshal`. The encoder emits nil slices as `[]`

internal/app/config_commands_test.go

@@ -207,6 +207,56 @@ scriptorium:
 	}
 }
 
+func TestConfigShowPreservesNullAndEmptyCollections(t *testing.T) {
+	dir := t.TempDir()
+	pipelinePath := filepath.Join(dir, "pipeline.yml")
+	mustWriteTestFile(t, pipelinePath, `workspace:
+  root: /tmp/narratio-work
+whisperx:
+  transcribe_url: https://transcription.example.test/transcribe
+audita:
+  modules: null
+scriptorium:
+  artifacts: null
+`)
+	mustWriteTestFile(t, filepath.Join(dir, "profiles", "empty.yml"), `audita:
+  modules: []
+scriptorium:
+  artifacts: {}
+`)
+
+	var base, empty bytes.Buffer
+	if err := ConfigShow(context.Background(), []string{"--config", pipelinePath}, &base); err != nil {
+		t.Fatal(err)
+	}
+	if err := ConfigShow(context.Background(), []string{"--config", pipelinePath, "--profile", "empty"}, &empty); err != nil {
+		t.Fatal(err)
+	}
+	for _, want := range []string{"modules: null\n", "artifacts: null\n"} {
+		if !strings.Contains(base.String(), want) {
+			t.Fatalf("base show output missing %q:\n%s", want, base.String())
+		}
+	}
+	for _, want := range []string{"modules: []\n", "artifacts: {}\n"} {
+		if !strings.Contains(empty.String(), want) {
+			t.Fatalf("empty-profile show output missing %q:\n%s", want, empty.String())
+		}
+	}
+
+	var diff bytes.Buffer
+	if err := ConfigDiff(context.Background(), []string{"default", "empty", "--config", pipelinePath}, &diff); err != nil {
+		t.Fatal(err)
+	}
+	for _, want := range []string{
+		"changed\taudita.modules\tnull\t[]\n",
+		"changed\tscriptorium.artifacts\tnull\t{}\n",
+	} {
+		if !strings.Contains(diff.String(), want) {
+			t.Fatalf("diff output missing %q:\n%s", want, diff.String())
+		}
+	}
+}
+
 func TestConfigDiffComparesExpandedFamiliesAndPublishRules(t *testing.T) {
 	pipelinePath, campaignPath := writeInspectionFamilyConfig(t)
 	writeInspectionProfiles(t, pipelinePath, `scriptorium:

internal/config/effective_pipeline.go

@@ -1,9 +1,11 @@
 package config
 
 import (
+	"encoding/json"
 	"fmt"
 	"path/filepath"
 	"sort"
+	"strconv"
 	"strings"
 
 	"gopkg.in/yaml.v3"
@@ -151,13 +153,72 @@ func MarshalEffectivePipeline(cfg *PipelineConfig) ([]byte, error) {
 		return nil, fmt.Errorf("pipeline config is required")
 	}
 
-	data, err := yaml.Marshal(projectEffectivePipeline(cfg, true))
+	value, err := projectionValueTree(projectEffectivePipeline(cfg, true))
+	if err != nil {
+		return nil, err
+	}
+	node, err := effectiveYAMLNode(value)
+	if err != nil {
+		return nil, fmt.Errorf("serialize effective pipeline: %w", err)
+	}
+	data, err := yaml.Marshal(node)
 	if err != nil {
 		return nil, fmt.Errorf("serialize effective pipeline: %w", err)
 	}
 	return data, nil
 }
 
+func effectiveYAMLNode(value any) (*yaml.Node, error) {
+	switch value := value.(type) {
+	case nil:
+		return &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!null", Value: "null"}, nil
+	case bool:
+		return &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!bool", Value: strconv.FormatBool(value)}, nil
+	case string:
+		return &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: value}, nil
+	case json.Number:
+		tag := "!!int"
+		if strings.ContainsAny(value.String(), ".eE") {
+			tag = "!!float"
+		}
+		return &yaml.Node{Kind: yaml.ScalarNode, Tag: tag, Value: value.String()}, nil
+	case []any:
+		node := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
+		if len(value) == 0 {
+			node.Style = yaml.FlowStyle
+		}
+		for _, item := range value {
+			child, err := effectiveYAMLNode(item)
+			if err != nil {
+				return nil, err
+			}
+			node.Content = append(node.Content, child)
+		}
+		return node, nil
+	case map[string]any:
+		node := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
+		if len(value) == 0 {
+			node.Style = yaml.FlowStyle
+		}
+		keys := make([]string, 0, len(value))
+		for key := range value {
+			keys = append(keys, key)
+		}
+		sort.Strings(keys)
+		for _, key := range keys {
+			child, err := effectiveYAMLNode(value[key])
+			if err != nil {
+				return nil, err
+			}
+			node.Content = append(node.Content,
+				&yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: key}, child)
+		}
+		return node, nil
+	default:
+		return nil, fmt.Errorf("unsupported projected value type %T", value)
+	}
+}
+
 func effectivePipelineOwners(cfg *PipelineConfig) map[string][]string {
 	owners := make(map[string][]string)
 	if cfg == nil || cfg.resolution == nil {

internal/config/effective_projection.go

@@ -74,6 +74,20 @@ func projectionJSON(projection effectivePipelineProjection) ([]byte, error) {
 	return data, nil
 }
 
+func projectionValueTree(value any) (any, error) {
+	data, err := json.Marshal(value)
+	if err != nil {
+		return nil, fmt.Errorf("encode typed pipeline projection: %w", err)
+	}
+	decoder := json.NewDecoder(bytes.NewReader(data))
+	decoder.UseNumber()
+	var root any
+	if err := decoder.Decode(&root); err != nil {
+		return nil, fmt.Errorf("decode effective pipeline projection: %w", err)
+	}
+	return root, nil
+}
+
 func projectionValueRecords(projection effectivePipelineProjection) ([]EffectivePipelineValueRecord, error) {
 	detailed, err := projectionDetailedRecords(projection)
 	if err != nil {
@@ -87,15 +101,13 @@ func projectionValueRecords(projection effectivePipelineProjection) ([]Effective
 }
 
 func projectionDetailedRecords(value any) ([]projectedPipelineValue, error) {
-	data, err := json.Marshal(value)
+	decoded, err := projectionValueTree(value)
 	if err != nil {
-		return nil, fmt.Errorf("encode typed pipeline projection: %w", err)
+		return nil, err
 	}
-	decoder := json.NewDecoder(bytes.NewReader(data))
+	root, ok := decoded.(map[string]any)
-	decoder.UseNumber()
+	if !ok {
-	var root map[string]any
+		return nil, fmt.Errorf("effective pipeline projection must be a mapping")
-	if err := decoder.Decode(&root); err != nil {
-		return nil, fmt.Errorf("decode effective pipeline projection: %w", err)
 	}
 	var records []projectedPipelineValue
 	appendProjectionRecords(&records, nil, root)

internal/config/load_validate_test.go

@@ -159,15 +159,44 @@ storage:
 }
 
 func TestEffectivePipelineValuesPreserveNilEmptyAndExactIntegers(t *testing.T) {
-	base := &PipelineConfig{Audita: AuditaConfig{Modules: nil}}
+	base := &PipelineConfig{
+		Audita:      AuditaConfig{Modules: nil},
+		Scriptorium: &ScriptoriumConfig{Artifacts: nil},
+		resolution:  &pipelineResolutionMetadata{},
+	}
 	nilValues, err := EffectivePipelineValues(base)
 	if err != nil {
 		t.Fatal(err)
 	}
+	nilYAML, err := MarshalEffectivePipeline(base)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(string(nilYAML), "modules: null\n") || !strings.Contains(string(nilYAML), "artifacts: null\n") {
+		t.Fatalf("effective YAML lost null collections: %q", nilYAML)
+	}
+	if err := recomputePipelineEffectiveDigest(base); err != nil {
+		t.Fatal(err)
+	}
+	nilDigest := EffectivePipelineDigest(base)
 	base.Audita.Modules = []string{}
-	base.Scriptorium = &ScriptoriumConfig{Artifacts: map[string]ScriptoriumArtifactConfig{
+	base.Scriptorium.Artifacts = map[string]ScriptoriumArtifactConfig{}
+	emptyYAML, err := MarshalEffectivePipeline(base)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(string(emptyYAML), "modules: []\n") || !strings.Contains(string(emptyYAML), "artifacts: {}\n") {
+		t.Fatalf("effective YAML lost empty collections: %q", emptyYAML)
+	}
+	if err := recomputePipelineEffectiveDigest(base); err != nil {
+		t.Fatal(err)
+	}
+	if emptyDigest := EffectivePipelineDigest(base); emptyDigest == nilDigest {
+		t.Fatalf("null and empty collections have equal digest %q", emptyDigest)
+	}
+	base.Scriptorium.Artifacts = map[string]ScriptoriumArtifactConfig{
 		"recap": {Vars: map[string]any{"exact": int64(9007199254740993)}},
-	}}
+	}
 	emptyValues, err := EffectivePipelineValues(base)
 	if err != nil {
 		t.Fatal(err)