notarius/internal/framework/pipeline/references.go

package pipeline

import (
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"net/url"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"unicode/utf8"

	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)

const (
	referenceOriginFile = "file"
	referenceMediaType  = "text/plain; charset=utf-8"
)

type ReferenceMaterializationOptions struct {
	ConfigPath string
	WorkingDir string
}

func MaterializeReferences(resolved ResolvedPipeline, catalog ModuleCatalog, options ReferenceMaterializationOptions) (ResolvedPipeline, []contracts.Warning, error) {
	out := resolved
	if len(resolved.ArtifactLanes) == 0 {
		return out, nil, nil
	}

	warnings := []contracts.Warning(nil)
	out.ArtifactLanes = make([]ResolvedArtifactLane, len(resolved.ArtifactLanes))
	for i, lane := range resolved.ArtifactLanes {
		materializedLane := lane
		referenceSet, laneWarnings, err := materializeLaneReferences(resolved.ID, lane, catalog, options)
		if err != nil {
			return ResolvedPipeline{}, nil, err
		}
		materializedLane.ReferenceSet = referenceSet
		out.ArtifactLanes[i] = materializedLane
		warnings = append(warnings, laneWarnings...)
	}
	return out, warnings, nil
}

func materializeLaneReferences(
	pipelineID string,
	lane ResolvedArtifactLane,
	catalog ModuleCatalog,
	options ReferenceMaterializationOptions,
) (contracts.ReferenceSet, []contracts.Warning, error) {
	if len(lane.References) == 0 {
		return contracts.ReferenceSet{}, nil, nil
	}
	if catalog.Extractors == nil {
		return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q extract module %q: module %q is not registered", pipelineID, lane.ID, lane.Extract.Module, lane.Extract.Module)
	}
	spec, ok := catalog.Extractors.Spec(lane.Extract.Module)
	if !ok {
		return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q extract module %q: module %q is not registered", pipelineID, lane.ID, lane.Extract.Module, lane.Extract.Module)
	}

	slotByName := make(map[string]contracts.ReferenceSlot, len(spec.ReferenceSlots))
	for _, slot := range spec.ReferenceSlots {
		slotByName[slot.Name] = slot
	}

	set := contracts.ReferenceSet{Slots: make(map[string]contracts.ResolvedReferenceSlot, len(lane.References))}
	var warnings []contracts.Warning
	for _, binding := range lane.References {
		slotName := strings.TrimSpace(binding.SlotName)
		slot, ok := slotByName[slotName]
		if !ok {
			return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q is not declared by extractor %q", pipelineID, lane.ID, slotName, lane.Extract.Module)
		}

		path, err := referencePath(binding, options)
		if err != nil {
			return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q path %q: %w", pipelineID, lane.ID, slotName, binding.Source, err)
		}
		content, err := os.ReadFile(path)
		if err != nil {
			return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q read %q: %w", pipelineID, lane.ID, slotName, path, err)
		}
		if !utf8.Valid(content) {
			return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q path %q must be UTF-8 text", pipelineID, lane.ID, slotName, path)
		}
		if slot.MaxBytes > 0 && int64(len(content)) > slot.MaxBytes {
			return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q path %q is %d bytes, limit %d", pipelineID, lane.ID, slotName, path, len(content), slot.MaxBytes)
		}
		if len(content) == 0 {
			warnings = append(warnings, contracts.Warning{
				Scope:      fmt.Sprintf("pipeline.%s.lane.%s.reference.%s", pipelineID, lane.ID, slotName),
				ReasonCode: "empty_reference",
				Message:    fmt.Sprintf("reference slot %q for lane %q is bound to an empty file", slotName, lane.ID),
			})
		}

		item := contracts.ReferenceItem{
			SlotName:      slotName,
			MediaType:     referenceMediaType,
			Content:       append([]byte(nil), content...),
			Digest:        referenceDigest(content),
			Origin:        contracts.ReferenceOrigin{Type: referenceOriginFile, URI: fileURI(path)},
			SizeBytes:     int64(len(content)),
			BindingSource: strings.TrimSpace(binding.BindingSource),
		}
		set.Slots[slotName] = contracts.ResolvedReferenceSlot{
			Slot:  cloneReferenceSlot(slot),
			Items: []contracts.ReferenceItem{item},
		}
	}
	return set, warnings, nil
}

func referencePath(binding ReferenceBinding, options ReferenceMaterializationOptions) (string, error) {
	source := strings.TrimSpace(binding.Source)
	if source == "" {
		return "", fmt.Errorf("must not be empty")
	}
	if filepath.IsAbs(source) {
		return filepath.Clean(source), nil
	}

	base := strings.TrimSpace(options.WorkingDir)
	if strings.TrimSpace(binding.BindingSource) == contracts.ReferenceBindingSourceConfig {
		base = filepath.Dir(strings.TrimSpace(options.ConfigPath))
	}
	if base == "" {
		var err error
		base, err = os.Getwd()
		if err != nil {
			return "", fmt.Errorf("resolve working directory: %w", err)
		}
	}
	return filepath.Clean(filepath.Join(base, source)), nil
}

func referenceDigest(content []byte) string {
	sum := sha256.Sum256(content)
	return "sha256:" + hex.EncodeToString(sum[:])
}

func fileURI(path string) string {
	absolute, err := filepath.Abs(path)
	if err != nil {
		absolute = path
	}
	absolute = filepath.ToSlash(filepath.Clean(absolute))
	if strings.HasPrefix(absolute, "/") {
		return "file://" + (&url.URL{Path: absolute}).EscapedPath()
	}
	return "file:///" + (&url.URL{Path: absolute}).EscapedPath()
}

func cloneReferenceSlot(slot contracts.ReferenceSlot) contracts.ReferenceSlot {
	slot.AcceptedMediaTypes = append([]string(nil), slot.AcceptedMediaTypes...)
	return slot
}

func CloneReferenceSet(in contracts.ReferenceSet) contracts.ReferenceSet {
	if len(in.Slots) == 0 {
		return contracts.ReferenceSet{}
	}
	out := contracts.ReferenceSet{Slots: make(map[string]contracts.ResolvedReferenceSlot, len(in.Slots))}
	keys := make([]string, 0, len(in.Slots))
	for key := range in.Slots {
		keys = append(keys, key)
	}
	sort.Strings(keys)
	for _, key := range keys {
		slot := in.Slots[key]
		slot.Slot = cloneReferenceSlot(slot.Slot)
		if len(slot.Items) > 0 {
			items := make([]contracts.ReferenceItem, len(slot.Items))
			for i, item := range slot.Items {
				item.Content = append([]byte(nil), item.Content...)
				items[i] = item
			}
			slot.Items = items
		}
		out.Slots[key] = slot
	}
	return out
}