promptkit/internal/catalog/catalog.go

// Package catalog validates immutable maintained backend catalog assets.
package catalog

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/fs"
	"path"
	"sort"
	"strings"

	"gitea.maximumdirect.net/eric/promptkit/internal/backend"
	"gitea.maximumdirect.net/eric/promptkit/internal/domain"
	"gitea.maximumdirect.net/eric/promptkit/internal/profile"
)

// Source identifies one immutable backend catalog asset tree.
type Source struct {
	Name              string
	ExpectedBackendID string
	FS                fs.FS
	Root              string
}

// Set is the validated maintained backend and raw profile catalog.
type Set struct {
	Backends   []domain.Backend
	Profiles   profile.Repository
	profileIDs []string
}

// Load validates and combines immutable catalog sources in source order.
func Load(sources ...Source) (Set, error) {
	if len(sources) == 0 {
		return Set{}, errors.New("at least one catalog source is required")
	}
	loaded := Set{Backends: make([]domain.Backend, 0, len(sources))}
	names := map[string]bool{}
	backendIDs := map[string]bool{}
	profileIDs := map[string]bool{}
	for _, source := range sources {
		if err := validateSource(source, names); err != nil {
			return Set{}, err
		}
		names[source.Name] = true
		definition, err := loadBackend(source)
		if err != nil {
			return Set{}, err
		}
		if backendIDs[definition.ID] {
			return Set{}, fmt.Errorf("catalog %s: backend ID duplicates an earlier catalog", source.Name)
		}
		backendIDs[definition.ID] = true
		repository, metadata, err := profile.LoadFSRepository(context.Background(), source.FS, path.Join(source.Root, "profiles"))
		if err != nil {
			return Set{}, fmt.Errorf("catalog %s: %w", source.Name, err)
		}
		if len(metadata) == 0 {
			return Set{}, fmt.Errorf("catalog %s: profiles must not be empty", source.Name)
		}
		resolvingRepository := profile.NewResolvingRepository(repository)
		for _, entry := range metadata {
			if profileIDs[entry.ID] {
				return Set{}, fmt.Errorf("catalog %s: %s duplicates an earlier profile ID", source.Name, entry.Path)
			}
			if containsField(entry.ExplicitFields, "endpoint") || containsField(entry.ExplicitFields, "api_key_env") {
				return Set{}, fmt.Errorf("catalog %s: %s contains connection metadata", source.Name, entry.Path)
			}
			value, err := repository.GetProfile(context.Background(), entry.ID)
			if err != nil {
				return Set{}, fmt.Errorf("catalog %s: %s: %w", source.Name, entry.Path, err)
			}
			if err := rejectSecretKeys(value.ExtraParams); err != nil {
				return Set{}, fmt.Errorf("catalog %s: %s: prohibited extra parameter key", source.Name, entry.Path)
			}
			resolved, err := resolvingRepository.GetProfile(context.Background(), entry.ID)
			if err != nil {
				return Set{}, fmt.Errorf("catalog %s: %s has invalid profile inheritance", source.Name, entry.Path)
			}
			if resolved.BackendID != definition.ID {
				return Set{}, fmt.Errorf("catalog %s: %s selects a different backend", source.Name, entry.Path)
			}
			profileIDs[entry.ID] = true
			loaded.profileIDs = append(loaded.profileIDs, entry.ID)
		}
		if loaded.Profiles == nil {
			loaded.Profiles = repository
		} else {
			loaded.Profiles = profile.NewOverlayRepository(loaded.Profiles, repository)
		}
		loaded.Backends = append(loaded.Backends, definition)
	}
	sort.Strings(loaded.profileIDs)
	return loaded, nil
}

func validateSource(source Source, names map[string]bool) error {
	if strings.TrimSpace(source.Name) == "" || source.Name != strings.TrimSpace(source.Name) {
		return errors.New("catalog source name must not be blank")
	}
	if names[source.Name] {
		return fmt.Errorf("duplicate catalog source name %q", source.Name)
	}
	if source.FS == nil {
		return fmt.Errorf("catalog %s: filesystem is nil", source.Name)
	}
	if source.Root == "." || source.Root == "" || !fs.ValidPath(source.Root) {
		return fmt.Errorf("catalog %s: asset root is invalid", source.Name)
	}
	if strings.TrimSpace(source.ExpectedBackendID) == "" {
		return fmt.Errorf("catalog %s: expected backend ID is blank", source.Name)
	}
	return validateLayout(source)
}

func validateLayout(source Source) error {
	manifestPath := path.Join(source.Root, "backend.json")
	profilesRoot := path.Join(source.Root, "profiles")
	return fs.WalkDir(source.FS, source.Root, func(assetPath string, entry fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if assetPath == source.Root {
			if !entry.IsDir() {
				return fmt.Errorf("catalog %s: invalid asset path %s", source.Name, assetPath)
			}
			return nil
		}
		if entry.IsDir() {
			if assetPath == profilesRoot || strings.HasPrefix(assetPath, profilesRoot+"/") {
				return nil
			}
			return fmt.Errorf("catalog %s: invalid asset path %s", source.Name, assetPath)
		}
		if assetPath == manifestPath && entry.Type().IsRegular() {
			return nil
		}
		if strings.HasPrefix(assetPath, profilesRoot+"/") && entry.Type().IsRegular() && strings.HasSuffix(assetPath, ".yml") {
			return nil
		}
		return fmt.Errorf("catalog %s: invalid asset path %s", source.Name, assetPath)
	})
}

func loadBackend(source Source) (domain.Backend, error) {
	data, err := fs.ReadFile(source.FS, path.Join(source.Root, "backend.json"))
	if err != nil {
		return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: %w", source.Name, err)
	}
	type manifest struct {
		SchemaVersion    *int            `json:"schema_version"`
		ID               *string         `json:"id"`
		Endpoint         *string         `json:"endpoint"`
		APIKeyEnv        *string         `json:"api_key_env"`
		ConcurrencyLimit *int            `json:"concurrency_limit"`
		QueueCapacity    *int            `json:"queue_capacity"`
		ExtraParams      json.RawMessage `json:"extra_params"`
	}
	decoder := json.NewDecoder(bytes.NewReader(data))
	decoder.DisallowUnknownFields()
	var value manifest
	if err := decoder.Decode(&value); err != nil {
		return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid manifest", source.Name)
	}
	var trailing any
	if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
		return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid manifest", source.Name)
	}
	if value.SchemaVersion == nil || *value.SchemaVersion != 1 || value.ID == nil || value.Endpoint == nil || value.APIKeyEnv == nil || value.ConcurrencyLimit == nil || value.QueueCapacity == nil || value.ExtraParams == nil {
		return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: required field is missing or unsupported", source.Name)
	}
	if *value.ID != source.ExpectedBackendID {
		return domain.Backend{}, fmt.Errorf("catalog %s: backend ID does not match expected ID", source.Name)
	}
	if strings.TrimSpace(*value.APIKeyEnv) == "" {
		return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: api key environment variable must not be blank", source.Name)
	}
	extraParams, err := decodeExtraParams(value.ExtraParams)
	if err != nil {
		return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid extra parameters", source.Name)
	}
	if err := rejectSecretKeys(extraParams); err != nil {
		return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: prohibited extra parameter key", source.Name)
	}
	normalized, err := backend.NormalizeDefinition(domain.Backend{ID: *value.ID, Endpoint: *value.Endpoint, APIKeyEnv: *value.APIKeyEnv, ExtraParams: extraParams, ConcurrencyLimit: *value.ConcurrencyLimit, QueueCapacity: *value.QueueCapacity, QueueCapacitySet: true})
	if err != nil {
		return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid backend definition", source.Name)
	}
	return normalized, nil
}

func decodeExtraParams(data []byte) (map[string]any, error) {
	if bytes.Equal(bytes.TrimSpace(data), []byte("null")) {
		return nil, nil
	}
	decoder := json.NewDecoder(bytes.NewReader(data))
	decoder.UseNumber()
	var value map[string]any
	if err := decoder.Decode(&value); err != nil || value == nil {
		return nil, errors.New("extra parameters must be an object or null")
	}
	var trailing any
	if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
		return nil, errors.New("extra parameters must contain exactly one JSON value")
	}
	return value, nil
}

func containsField(fields []string, target string) bool {
	for _, field := range fields {
		if field == target {
			return true
		}
	}
	return false
}

func rejectSecretKeys(value map[string]any) error {
	return rejectSecretValue(value)
}

func rejectSecretValue(value any) error {
	switch value := value.(type) {
	case map[string]any:
		for key, child := range value {
			switch strings.ToLower(key) {
			case "api_key", "apikey", "authorization", "credential", "credentials", "password", "secret", "token", "access_token":
				return errors.New("prohibited key")
			}
			if err := rejectSecretValue(child); err != nil {
				return err
			}
		}
	case []any:
		for _, child := range value {
			if err := rejectSecretValue(child); err != nil {
				return err
			}
		}
	}
	return nil
}