Materialize extraction reference files

internal/framework/pipeline/references_test.go

@@ -0,0 +1,178 @@
+package pipeline
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
+)
+
+func TestMaterializeReferencesResolvesPathsAndDigestsContent(t *testing.T) {
+	configDir := t.TempDir()
+	workingDir := t.TempDir()
+	configReference := filepath.Join(configDir, "config-reference.txt")
+	cliReference := filepath.Join(workingDir, "cli-reference.txt")
+	writeReferenceFile(t, configReference, []byte("config text"))
+	writeReferenceFile(t, cliReference, []byte("cli text"))
+
+	pipeline := baselineProfile()
+	pipeline.References = map[string]string{"roster": "config-reference.txt"}
+	lane := pipeline.Artifacts["events"]
+	lane.References = map[string]string{"glossary": "cli-reference.txt"}
+	pipeline.Artifacts["events"] = lane
+	catalog := referenceCatalog(t, []contracts.ReferenceSlot{
+		{Name: "roster"},
+		{Name: "glossary"},
+	})
+	resolved, err := ResolvePipeline(pipeline, ResolveOptions{
+		ReferenceOverrides: []ReferenceBinding{
+			{LaneID: "events", SlotName: "glossary", Source: "cli-reference.txt", BindingSource: contracts.ReferenceBindingSourceCLI},
+		},
+	}, catalog)
+	if err != nil {
+		t.Fatalf("ResolvePipeline() error = %v, want nil", err)
+	}
+
+	first, warnings, err := MaterializeReferences(resolved, catalog, ReferenceMaterializationOptions{
+		ConfigPath: filepath.Join(configDir, "config.yml"),
+		WorkingDir: workingDir,
+	})
+	if err != nil {
+		t.Fatalf("MaterializeReferences() error = %v, want nil", err)
+	}
+	if len(warnings) != 0 {
+		t.Fatalf("warnings = %#v, want none", warnings)
+	}
+	second, _, err := MaterializeReferences(resolved, catalog, ReferenceMaterializationOptions{
+		ConfigPath: filepath.Join(configDir, "config.yml"),
+		WorkingDir: workingDir,
+	})
+	if err != nil {
+		t.Fatalf("MaterializeReferences(second) error = %v, want nil", err)
+	}
+
+	referenceSet := first.ArtifactLanes[0].ReferenceSet
+	roster := referenceSet.Slots["roster"].Items[0]
+	if string(roster.Content) != "config text" {
+		t.Fatalf("roster content = %q, want config text", roster.Content)
+	}
+	if roster.Digest != referenceDigest([]byte("config text")) || roster.Digest != second.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0].Digest {
+		t.Fatalf("roster digest = %q, want stable digest", roster.Digest)
+	}
+	if roster.BindingSource != contracts.ReferenceBindingSourceConfig {
+		t.Fatalf("roster binding source = %q, want config", roster.BindingSource)
+	}
+	if roster.MediaType != referenceMediaType || roster.Origin.Type != referenceOriginFile || roster.SizeBytes != int64(len("config text")) {
+		t.Fatalf("roster metadata = %#v, want text file metadata", roster)
+	}
+	if !strings.Contains(roster.Origin.URI, "config-reference.txt") {
+		t.Fatalf("roster origin URI = %q, want config reference path", roster.Origin.URI)
+	}
+
+	glossary := referenceSet.Slots["glossary"].Items[0]
+	if string(glossary.Content) != "cli text" {
+		t.Fatalf("glossary content = %q, want cli text", glossary.Content)
+	}
+	if glossary.BindingSource != contracts.ReferenceBindingSourceCLI {
+		t.Fatalf("glossary binding source = %q, want cli", glossary.BindingSource)
+	}
+	if !strings.Contains(glossary.Origin.URI, "cli-reference.txt") {
+		t.Fatalf("glossary origin URI = %q, want cli reference path", glossary.Origin.URI)
+	}
+
+	encoded, err := json.Marshal(first)
+	if err != nil {
+		t.Fatalf("json.Marshal(materialized) error = %v, want nil", err)
+	}
+	if strings.Contains(string(encoded), "config text") || strings.Contains(string(encoded), "cli text") {
+		t.Fatalf("materialized pipeline JSON contains reference content: %s", encoded)
+	}
+}
+
+func TestMaterializeReferencesRejectsNonUTF8Content(t *testing.T) {
+	configDir := t.TempDir()
+	path := filepath.Join(configDir, "bad.txt")
+	writeReferenceFile(t, path, []byte{0xff, 0xfe})
+
+	resolved := resolvedPipelineWithReference(t, "roster", "bad.txt", contracts.ReferenceBindingSourceConfig, contracts.ReferenceSlot{Name: "roster"})
+	_, _, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{{Name: "roster"}}), ReferenceMaterializationOptions{
+		ConfigPath: filepath.Join(configDir, "config.yml"),
+	})
+	if err == nil || !strings.Contains(err.Error(), "UTF-8") || !strings.Contains(err.Error(), "roster") || !strings.Contains(err.Error(), path) {
+		t.Fatalf("error = %v, want UTF-8 path error", err)
+	}
+}
+
+func TestMaterializeReferencesWarnsForEmptyFiles(t *testing.T) {
+	configDir := t.TempDir()
+	path := filepath.Join(configDir, "empty.txt")
+	writeReferenceFile(t, path, nil)
+
+	resolved := resolvedPipelineWithReference(t, "roster", "empty.txt", contracts.ReferenceBindingSourceConfig, contracts.ReferenceSlot{Name: "roster"})
+	materialized, warnings, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{{Name: "roster"}}), ReferenceMaterializationOptions{
+		ConfigPath: filepath.Join(configDir, "config.yml"),
+	})
+	if err != nil {
+		t.Fatalf("MaterializeReferences() error = %v, want nil", err)
+	}
+	if len(warnings) != 1 || warnings[0].ReasonCode != "empty_reference" {
+		t.Fatalf("warnings = %#v, want empty reference warning", warnings)
+	}
+	item := materialized.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0]
+	if item.SizeBytes != 0 || item.Digest != referenceDigest(nil) {
+		t.Fatalf("empty item = %#v, want zero size and empty digest", item)
+	}
+}
+
+func TestMaterializeReferencesEnforcesMaxBytes(t *testing.T) {
+	configDir := t.TempDir()
+	path := filepath.Join(configDir, "large.txt")
+	writeReferenceFile(t, path, []byte("too large"))
+
+	slot := contracts.ReferenceSlot{Name: "roster", MaxBytes: 3}
+	resolved := resolvedPipelineWithReference(t, "roster", "large.txt", contracts.ReferenceBindingSourceConfig, slot)
+	_, _, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{slot}), ReferenceMaterializationOptions{
+		ConfigPath: filepath.Join(configDir, "config.yml"),
+	})
+	if err == nil || !strings.Contains(err.Error(), "9 bytes") || !strings.Contains(err.Error(), "limit 3") || !strings.Contains(err.Error(), "roster") {
+		t.Fatalf("error = %v, want max bytes error", err)
+	}
+}
+
+func resolvedPipelineWithReference(t *testing.T, slotName, source, bindingSource string, slot contracts.ReferenceSlot) ResolvedPipeline {
+	t.Helper()
+	profile := baselineProfile()
+	lane := profile.Artifacts["events"]
+	lane.References = map[string]string{slotName: source}
+	profile.Artifacts["events"] = lane
+	catalog := referenceCatalog(t, []contracts.ReferenceSlot{slot})
+	resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
+	if err != nil {
+		t.Fatalf("ResolvePipeline() error = %v, want nil", err)
+	}
+	if bindingSource != contracts.ReferenceBindingSourceConfig {
+		resolved.ArtifactLanes[0].References[0].BindingSource = bindingSource
+	}
+	return resolved
+}
+
+func referenceCatalog(t *testing.T, slots []contracts.ReferenceSlot) ModuleCatalog {
+	t.Helper()
+	return newProfileCatalogWithOverride(t, ModuleSpec{
+		Key:            "event-extractor",
+		Stage:          StageExtract,
+		Requires:       []string{"chunk"},
+		Provides:       []string{"candidate"},
+		ReferenceSlots: slots,
+	})
+}
+
+func writeReferenceFile(t *testing.T, path string, content []byte) {
+	t.Helper()
+	if err := os.WriteFile(path, content, 0o644); err != nil {
+		t.Fatalf("write reference %q: %v", path, err)
+	}
+}