280 lines
12 KiB
Go
280 lines
12 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"encoding/json"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
)
|
|
|
|
func TestMaterializeReferencesResolvesPathsAndDigestsContent(t *testing.T) {
|
|
configDir := t.TempDir()
|
|
workingDir := t.TempDir()
|
|
configReference := filepath.Join(configDir, "config-reference.txt")
|
|
cliReference := filepath.Join(workingDir, "cli-reference.txt")
|
|
writeReferenceFile(t, configReference, []byte("config text"))
|
|
writeReferenceFile(t, cliReference, []byte("cli text"))
|
|
|
|
pipeline := baselineProfile()
|
|
pipeline.References = map[string]string{"roster": "config-reference.txt"}
|
|
lane := pipeline.Artifacts["events"]
|
|
lane.References = map[string]string{"glossary": "cli-reference.txt"}
|
|
pipeline.Artifacts["events"] = lane
|
|
catalog := referenceCatalog(t, []contracts.ReferenceSlot{
|
|
{Name: "roster"},
|
|
{Name: "glossary"},
|
|
})
|
|
resolved, err := ResolvePipeline(pipeline, ResolveOptions{
|
|
ReferenceOverrides: []ReferenceBinding{
|
|
{LaneID: "events", SlotName: "glossary", Source: "cli-reference.txt", BindingSource: contracts.ReferenceBindingSourceCLI},
|
|
},
|
|
}, catalog)
|
|
if err != nil {
|
|
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
|
}
|
|
|
|
first, warnings, err := MaterializeReferences(resolved, catalog, ReferenceMaterializationOptions{
|
|
ConfigPath: filepath.Join(configDir, "config.yml"),
|
|
WorkingDir: workingDir,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
|
|
}
|
|
if len(warnings) != 0 {
|
|
t.Fatalf("warnings = %#v, want none", warnings)
|
|
}
|
|
second, _, err := MaterializeReferences(resolved, catalog, ReferenceMaterializationOptions{
|
|
ConfigPath: filepath.Join(configDir, "config.yml"),
|
|
WorkingDir: workingDir,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("MaterializeReferences(second) error = %v, want nil", err)
|
|
}
|
|
|
|
referenceSet := first.ArtifactLanes[0].ReferenceSet
|
|
roster := referenceSet.Slots["roster"].Items[0]
|
|
if string(roster.Content) != "config text" {
|
|
t.Fatalf("roster content = %q, want config text", roster.Content)
|
|
}
|
|
if roster.Digest != referenceDigest([]byte("config text")) || roster.Digest != second.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0].Digest {
|
|
t.Fatalf("roster digest = %q, want stable digest", roster.Digest)
|
|
}
|
|
if roster.BindingSource != contracts.ReferenceBindingSourceConfig {
|
|
t.Fatalf("roster binding source = %q, want config", roster.BindingSource)
|
|
}
|
|
if roster.MediaType != referenceMediaType || roster.Origin.Type != referenceOriginFile || roster.SizeBytes != int64(len("config text")) {
|
|
t.Fatalf("roster metadata = %#v, want text file metadata", roster)
|
|
}
|
|
if !strings.Contains(roster.Origin.URI, "config-reference.txt") {
|
|
t.Fatalf("roster origin URI = %q, want config reference path", roster.Origin.URI)
|
|
}
|
|
|
|
glossary := referenceSet.Slots["glossary"].Items[0]
|
|
if string(glossary.Content) != "cli text" {
|
|
t.Fatalf("glossary content = %q, want cli text", glossary.Content)
|
|
}
|
|
if glossary.BindingSource != contracts.ReferenceBindingSourceCLI {
|
|
t.Fatalf("glossary binding source = %q, want cli", glossary.BindingSource)
|
|
}
|
|
if !strings.Contains(glossary.Origin.URI, "cli-reference.txt") {
|
|
t.Fatalf("glossary origin URI = %q, want cli reference path", glossary.Origin.URI)
|
|
}
|
|
|
|
provenance := ReferenceProvenance(first)
|
|
if len(provenance) != 2 {
|
|
t.Fatalf("ReferenceProvenance() = %#v, want two entries", provenance)
|
|
}
|
|
if provenance[0].LaneID != "events" || provenance[0].SlotName != "glossary" || provenance[0].Digest != glossary.Digest {
|
|
t.Fatalf("ReferenceProvenance()[0] = %#v, want sorted glossary provenance", provenance[0])
|
|
}
|
|
if provenance[1].LaneID != "events" || provenance[1].SlotName != "roster" || provenance[1].Digest != roster.Digest {
|
|
t.Fatalf("ReferenceProvenance()[1] = %#v, want roster provenance", provenance[1])
|
|
}
|
|
|
|
encoded, err := json.Marshal(first)
|
|
if err != nil {
|
|
t.Fatalf("json.Marshal(materialized) error = %v, want nil", err)
|
|
}
|
|
if strings.Contains(string(encoded), "config text") || strings.Contains(string(encoded), "cli text") {
|
|
t.Fatalf("materialized pipeline JSON contains reference content: %s", encoded)
|
|
}
|
|
}
|
|
|
|
func TestMaterializeReferencesRejectsNonUTF8Content(t *testing.T) {
|
|
configDir := t.TempDir()
|
|
path := filepath.Join(configDir, "bad.txt")
|
|
writeReferenceFile(t, path, []byte{0xff, 0xfe})
|
|
|
|
resolved := resolvedPipelineWithReference(t, "roster", "bad.txt", contracts.ReferenceBindingSourceConfig, contracts.ReferenceSlot{Name: "roster"})
|
|
_, _, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{{Name: "roster"}}), ReferenceMaterializationOptions{
|
|
ConfigPath: filepath.Join(configDir, "config.yml"),
|
|
})
|
|
if err == nil || !strings.Contains(err.Error(), "UTF-8") || !strings.Contains(err.Error(), "roster") || !strings.Contains(err.Error(), path) {
|
|
t.Fatalf("error = %v, want UTF-8 path error", err)
|
|
}
|
|
}
|
|
|
|
func TestMaterializeReferencesAllowsAnyMediaTypeWhenSlotDoesNotRestrictIt(t *testing.T) {
|
|
configDir := t.TempDir()
|
|
path := filepath.Join(configDir, "roster.reference")
|
|
writeReferenceFile(t, path, []byte("plain text"))
|
|
|
|
resolved := resolvedPipelineWithReference(t, "roster", "roster.reference", contracts.ReferenceBindingSourceConfig, contracts.ReferenceSlot{Name: "roster"})
|
|
materialized, _, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{{Name: "roster"}}), ReferenceMaterializationOptions{
|
|
ConfigPath: filepath.Join(configDir, "config.yml"),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
|
|
}
|
|
item := materialized.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0]
|
|
if item.MediaType != unknownMediaType {
|
|
t.Fatalf("MediaType = %q, want %q", item.MediaType, unknownMediaType)
|
|
}
|
|
}
|
|
|
|
func TestMaterializeReferencesAcceptsDeclaredMarkdownMediaType(t *testing.T) {
|
|
configDir := t.TempDir()
|
|
path := filepath.Join(configDir, "glossary.md")
|
|
writeReferenceFile(t, path, []byte("# Terms\n"))
|
|
|
|
slot := contracts.ReferenceSlot{Name: "glossary", AcceptedMediaTypes: []string{"text/markdown"}}
|
|
resolved := resolvedPipelineWithReference(t, "glossary", "glossary.md", contracts.ReferenceBindingSourceConfig, slot)
|
|
materialized, _, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{slot}), ReferenceMaterializationOptions{
|
|
ConfigPath: filepath.Join(configDir, "config.yml"),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
|
|
}
|
|
item := materialized.ArtifactLanes[0].ReferenceSet.Slots["glossary"].Items[0]
|
|
if item.MediaType != "text/markdown" {
|
|
t.Fatalf("MediaType = %q, want text/markdown", item.MediaType)
|
|
}
|
|
}
|
|
|
|
func TestMaterializeReferencesAcceptsDeclaredJSONMediaType(t *testing.T) {
|
|
configDir := t.TempDir()
|
|
path := filepath.Join(configDir, "roster.json")
|
|
writeReferenceFile(t, path, []byte(`{"aria":"cleric"}`))
|
|
|
|
slot := contracts.ReferenceSlot{Name: "roster", AcceptedMediaTypes: []string{"application/json"}}
|
|
resolved := resolvedPipelineWithReference(t, "roster", "roster.json", contracts.ReferenceBindingSourceConfig, slot)
|
|
materialized, _, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{slot}), ReferenceMaterializationOptions{
|
|
ConfigPath: filepath.Join(configDir, "config.yml"),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
|
|
}
|
|
item := materialized.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0]
|
|
if item.MediaType != "application/json" {
|
|
t.Fatalf("MediaType = %q, want application/json", item.MediaType)
|
|
}
|
|
}
|
|
|
|
func TestMaterializeReferencesRejectsUnacceptedMediaType(t *testing.T) {
|
|
configDir := t.TempDir()
|
|
path := filepath.Join(configDir, "roster.json")
|
|
writeReferenceFile(t, path, []byte(`{"aria":"cleric"}`))
|
|
|
|
slot := contracts.ReferenceSlot{Name: "roster", AcceptedMediaTypes: []string{"text/markdown"}}
|
|
resolved := resolvedPipelineWithReference(t, "roster", "roster.json", contracts.ReferenceBindingSourceConfig, slot)
|
|
_, _, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{slot}), ReferenceMaterializationOptions{
|
|
ConfigPath: filepath.Join(configDir, "config.yml"),
|
|
})
|
|
if err == nil || !strings.Contains(err.Error(), "media type") || !strings.Contains(err.Error(), "application/json") || !strings.Contains(err.Error(), "roster") {
|
|
t.Fatalf("error = %v, want media type rejection", err)
|
|
}
|
|
}
|
|
|
|
func TestMaterializeReferencesMatchesAcceptedMediaTypesIgnoringParameters(t *testing.T) {
|
|
configDir := t.TempDir()
|
|
path := filepath.Join(configDir, "roster.txt")
|
|
writeReferenceFile(t, path, []byte("Aria\n"))
|
|
|
|
slot := contracts.ReferenceSlot{Name: "roster", AcceptedMediaTypes: []string{"text/plain; charset=utf-8"}}
|
|
resolved := resolvedPipelineWithReference(t, "roster", "roster.txt", contracts.ReferenceBindingSourceConfig, slot)
|
|
materialized, _, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{slot}), ReferenceMaterializationOptions{
|
|
ConfigPath: filepath.Join(configDir, "config.yml"),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
|
|
}
|
|
item := materialized.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0]
|
|
if item.MediaType != referenceMediaType {
|
|
t.Fatalf("MediaType = %q, want %q", item.MediaType, referenceMediaType)
|
|
}
|
|
}
|
|
|
|
func TestMaterializeReferencesWarnsForEmptyFiles(t *testing.T) {
|
|
configDir := t.TempDir()
|
|
path := filepath.Join(configDir, "empty.txt")
|
|
writeReferenceFile(t, path, nil)
|
|
|
|
resolved := resolvedPipelineWithReference(t, "roster", "empty.txt", contracts.ReferenceBindingSourceConfig, contracts.ReferenceSlot{Name: "roster"})
|
|
materialized, warnings, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{{Name: "roster"}}), ReferenceMaterializationOptions{
|
|
ConfigPath: filepath.Join(configDir, "config.yml"),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
|
|
}
|
|
if len(warnings) != 1 || warnings[0].ReasonCode != "empty_reference" {
|
|
t.Fatalf("warnings = %#v, want empty reference warning", warnings)
|
|
}
|
|
item := materialized.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0]
|
|
if item.SizeBytes != 0 || item.Digest != referenceDigest(nil) {
|
|
t.Fatalf("empty item = %#v, want zero size and empty digest", item)
|
|
}
|
|
}
|
|
|
|
func TestMaterializeReferencesEnforcesMaxBytes(t *testing.T) {
|
|
configDir := t.TempDir()
|
|
path := filepath.Join(configDir, "large.txt")
|
|
writeReferenceFile(t, path, []byte("too large"))
|
|
|
|
slot := contracts.ReferenceSlot{Name: "roster", MaxBytes: 3}
|
|
resolved := resolvedPipelineWithReference(t, "roster", "large.txt", contracts.ReferenceBindingSourceConfig, slot)
|
|
_, _, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{slot}), ReferenceMaterializationOptions{
|
|
ConfigPath: filepath.Join(configDir, "config.yml"),
|
|
})
|
|
if err == nil || !strings.Contains(err.Error(), "9 bytes") || !strings.Contains(err.Error(), "limit 3") || !strings.Contains(err.Error(), "roster") {
|
|
t.Fatalf("error = %v, want max bytes error", err)
|
|
}
|
|
}
|
|
|
|
func resolvedPipelineWithReference(t *testing.T, slotName, source, bindingSource string, slot contracts.ReferenceSlot) ResolvedPipeline {
|
|
t.Helper()
|
|
profile := baselineProfile()
|
|
lane := profile.Artifacts["events"]
|
|
lane.References = map[string]string{slotName: source}
|
|
profile.Artifacts["events"] = lane
|
|
catalog := referenceCatalog(t, []contracts.ReferenceSlot{slot})
|
|
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
|
if err != nil {
|
|
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
|
}
|
|
if bindingSource != contracts.ReferenceBindingSourceConfig {
|
|
resolved.ArtifactLanes[0].References[0].BindingSource = bindingSource
|
|
}
|
|
return resolved
|
|
}
|
|
|
|
func referenceCatalog(t *testing.T, slots []contracts.ReferenceSlot) ModuleCatalog {
|
|
t.Helper()
|
|
return newProfileCatalogWithOverride(t, ModuleSpec{
|
|
Key: "event-extractor",
|
|
Stage: StageExtract,
|
|
Requires: []string{"chunk"},
|
|
Provides: []string{"candidate"},
|
|
ReferenceSlots: slots,
|
|
})
|
|
}
|
|
|
|
func writeReferenceFile(t *testing.T, path string, content []byte) {
|
|
t.Helper()
|
|
if err := os.WriteFile(path, content, 0o644); err != nil {
|
|
t.Fatalf("write reference %q: %v", path, err)
|
|
}
|
|
}
|