diff --git a/docs/roadmap/remediation.md b/docs/roadmap/remediation.md index 5c66ffa..09c6d66 100644 --- a/docs/roadmap/remediation.md +++ b/docs/roadmap/remediation.md @@ -128,7 +128,13 @@ valid overlay remains accepted. ## 4. Preserve null versus empty collections in effective YAML -Status: Open — P2 +Status: Resolved — P2 + +Resolution: Effective YAML is now emitted from the credential-safe JSON value +projection through an output-only YAML-node conversion. Explicit nulls, empty maps, +empty slices, booleans, strings, and exact JSON numbers retain their semantic types. +Public show/diff tests cover null and empty maps and slices, while digest, +credential-redaction, deterministic output, and exact-integer coverage remain intact. **Evidence:** `internal/config/effective_pipeline.go:154` marshals the typed projection directly with `yaml.Marshal`. The encoder emits nil slices as `[]` diff --git a/internal/app/config_commands_test.go b/internal/app/config_commands_test.go index 4cf1aaa..ba879be 100644 --- a/internal/app/config_commands_test.go +++ b/internal/app/config_commands_test.go @@ -207,6 +207,56 @@ scriptorium: } } +func TestConfigShowPreservesNullAndEmptyCollections(t *testing.T) { + dir := t.TempDir() + pipelinePath := filepath.Join(dir, "pipeline.yml") + mustWriteTestFile(t, pipelinePath, `workspace: + root: /tmp/narratio-work +whisperx: + transcribe_url: https://transcription.example.test/transcribe +audita: + modules: null +scriptorium: + artifacts: null +`) + mustWriteTestFile(t, filepath.Join(dir, "profiles", "empty.yml"), `audita: + modules: [] +scriptorium: + artifacts: {} +`) + + var base, empty bytes.Buffer + if err := ConfigShow(context.Background(), []string{"--config", pipelinePath}, &base); err != nil { + t.Fatal(err) + } + if err := ConfigShow(context.Background(), []string{"--config", pipelinePath, "--profile", "empty"}, &empty); err != nil { + t.Fatal(err) + } + for _, want := range []string{"modules: null\n", "artifacts: null\n"} { + if !strings.Contains(base.String(), want) { + t.Fatalf("base show output missing %q:\n%s", want, base.String()) + } + } + for _, want := range []string{"modules: []\n", "artifacts: {}\n"} { + if !strings.Contains(empty.String(), want) { + t.Fatalf("empty-profile show output missing %q:\n%s", want, empty.String()) + } + } + + var diff bytes.Buffer + if err := ConfigDiff(context.Background(), []string{"default", "empty", "--config", pipelinePath}, &diff); err != nil { + t.Fatal(err) + } + for _, want := range []string{ + "changed\taudita.modules\tnull\t[]\n", + "changed\tscriptorium.artifacts\tnull\t{}\n", + } { + if !strings.Contains(diff.String(), want) { + t.Fatalf("diff output missing %q:\n%s", want, diff.String()) + } + } +} + func TestConfigDiffComparesExpandedFamiliesAndPublishRules(t *testing.T) { pipelinePath, campaignPath := writeInspectionFamilyConfig(t) writeInspectionProfiles(t, pipelinePath, `scriptorium: diff --git a/internal/config/effective_pipeline.go b/internal/config/effective_pipeline.go index 85cb196..2b31a24 100644 --- a/internal/config/effective_pipeline.go +++ b/internal/config/effective_pipeline.go @@ -1,9 +1,11 @@ package config import ( + "encoding/json" "fmt" "path/filepath" "sort" + "strconv" "strings" "gopkg.in/yaml.v3" @@ -151,13 +153,72 @@ func MarshalEffectivePipeline(cfg *PipelineConfig) ([]byte, error) { return nil, fmt.Errorf("pipeline config is required") } - data, err := yaml.Marshal(projectEffectivePipeline(cfg, true)) + value, err := projectionValueTree(projectEffectivePipeline(cfg, true)) + if err != nil { + return nil, err + } + node, err := effectiveYAMLNode(value) + if err != nil { + return nil, fmt.Errorf("serialize effective pipeline: %w", err) + } + data, err := yaml.Marshal(node) if err != nil { return nil, fmt.Errorf("serialize effective pipeline: %w", err) } return data, nil } +func effectiveYAMLNode(value any) (*yaml.Node, error) { + switch value := value.(type) { + case nil: + return &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!null", Value: "null"}, nil + case bool: + return &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!bool", Value: strconv.FormatBool(value)}, nil + case string: + return &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: value}, nil + case json.Number: + tag := "!!int" + if strings.ContainsAny(value.String(), ".eE") { + tag = "!!float" + } + return &yaml.Node{Kind: yaml.ScalarNode, Tag: tag, Value: value.String()}, nil + case []any: + node := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"} + if len(value) == 0 { + node.Style = yaml.FlowStyle + } + for _, item := range value { + child, err := effectiveYAMLNode(item) + if err != nil { + return nil, err + } + node.Content = append(node.Content, child) + } + return node, nil + case map[string]any: + node := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"} + if len(value) == 0 { + node.Style = yaml.FlowStyle + } + keys := make([]string, 0, len(value)) + for key := range value { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + child, err := effectiveYAMLNode(value[key]) + if err != nil { + return nil, err + } + node.Content = append(node.Content, + &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: key}, child) + } + return node, nil + default: + return nil, fmt.Errorf("unsupported projected value type %T", value) + } +} + func effectivePipelineOwners(cfg *PipelineConfig) map[string][]string { owners := make(map[string][]string) if cfg == nil || cfg.resolution == nil { diff --git a/internal/config/effective_projection.go b/internal/config/effective_projection.go index 4b43dbe..4e43389 100644 --- a/internal/config/effective_projection.go +++ b/internal/config/effective_projection.go @@ -74,6 +74,20 @@ func projectionJSON(projection effectivePipelineProjection) ([]byte, error) { return data, nil } +func projectionValueTree(value any) (any, error) { + data, err := json.Marshal(value) + if err != nil { + return nil, fmt.Errorf("encode typed pipeline projection: %w", err) + } + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.UseNumber() + var root any + if err := decoder.Decode(&root); err != nil { + return nil, fmt.Errorf("decode effective pipeline projection: %w", err) + } + return root, nil +} + func projectionValueRecords(projection effectivePipelineProjection) ([]EffectivePipelineValueRecord, error) { detailed, err := projectionDetailedRecords(projection) if err != nil { @@ -87,15 +101,13 @@ func projectionValueRecords(projection effectivePipelineProjection) ([]Effective } func projectionDetailedRecords(value any) ([]projectedPipelineValue, error) { - data, err := json.Marshal(value) + decoded, err := projectionValueTree(value) if err != nil { - return nil, fmt.Errorf("encode typed pipeline projection: %w", err) + return nil, err } - decoder := json.NewDecoder(bytes.NewReader(data)) - decoder.UseNumber() - var root map[string]any - if err := decoder.Decode(&root); err != nil { - return nil, fmt.Errorf("decode effective pipeline projection: %w", err) + root, ok := decoded.(map[string]any) + if !ok { + return nil, fmt.Errorf("effective pipeline projection must be a mapping") } var records []projectedPipelineValue appendProjectionRecords(&records, nil, root) diff --git a/internal/config/load_validate_test.go b/internal/config/load_validate_test.go index 2c128a8..4a8f249 100644 --- a/internal/config/load_validate_test.go +++ b/internal/config/load_validate_test.go @@ -159,15 +159,44 @@ storage: } func TestEffectivePipelineValuesPreserveNilEmptyAndExactIntegers(t *testing.T) { - base := &PipelineConfig{Audita: AuditaConfig{Modules: nil}} + base := &PipelineConfig{ + Audita: AuditaConfig{Modules: nil}, + Scriptorium: &ScriptoriumConfig{Artifacts: nil}, + resolution: &pipelineResolutionMetadata{}, + } nilValues, err := EffectivePipelineValues(base) if err != nil { t.Fatal(err) } + nilYAML, err := MarshalEffectivePipeline(base) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(nilYAML), "modules: null\n") || !strings.Contains(string(nilYAML), "artifacts: null\n") { + t.Fatalf("effective YAML lost null collections: %q", nilYAML) + } + if err := recomputePipelineEffectiveDigest(base); err != nil { + t.Fatal(err) + } + nilDigest := EffectivePipelineDigest(base) base.Audita.Modules = []string{} - base.Scriptorium = &ScriptoriumConfig{Artifacts: map[string]ScriptoriumArtifactConfig{ + base.Scriptorium.Artifacts = map[string]ScriptoriumArtifactConfig{} + emptyYAML, err := MarshalEffectivePipeline(base) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(emptyYAML), "modules: []\n") || !strings.Contains(string(emptyYAML), "artifacts: {}\n") { + t.Fatalf("effective YAML lost empty collections: %q", emptyYAML) + } + if err := recomputePipelineEffectiveDigest(base); err != nil { + t.Fatal(err) + } + if emptyDigest := EffectivePipelineDigest(base); emptyDigest == nilDigest { + t.Fatalf("null and empty collections have equal digest %q", emptyDigest) + } + base.Scriptorium.Artifacts = map[string]ScriptoriumArtifactConfig{ "recap": {Vars: map[string]any{"exact": int64(9007199254740993)}}, - }} + } emptyValues, err := EffectivePipelineValues(base) if err != nil { t.Fatal(err)